From 90b767c3ec75817b1b98fdd81ad2034970096ef7 Mon Sep 17 00:00:00 2001 From: Alex Boten Date: Mon, 13 Dec 2021 15:49:11 -0800 Subject: [PATCH 01/31] `opentelemetry-exporter-prometheus`: restore package The Prometheus exporter was moved into the `metrics` branch prior to releasing 1.0 for tracing. Now that the metrics API and SDK are further along, we can bring back the exporter and update it to use the new methods. The original code for this PR comes from https://github.com/open-telemetry/opentelemetry-python/tree/metrics/exporter/opentelemetry-exporter-prometheus --- .../opentelemetry-exporter-prometheus/LICENSE | 201 ++++++++++++++++++ .../MANIFEST.in | 9 + .../README.rst | 23 ++ .../setup.cfg | 55 +++++ .../setup.py | 26 +++ .../exporter/prometheus/__init__.py | 193 +++++++++++++++++ .../exporter/prometheus/version.py | 15 ++ .../tests/__init__.py | 13 ++ .../tests/test_prometheus_exporter.py | 169 +++++++++++++++ 9 files changed, 704 insertions(+) create mode 100644 exporter/opentelemetry-exporter-prometheus/LICENSE create mode 100644 exporter/opentelemetry-exporter-prometheus/MANIFEST.in create mode 100644 exporter/opentelemetry-exporter-prometheus/README.rst create mode 100644 exporter/opentelemetry-exporter-prometheus/setup.cfg create mode 100644 exporter/opentelemetry-exporter-prometheus/setup.py create mode 100644 exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py create mode 100644 exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/version.py create mode 100644 exporter/opentelemetry-exporter-prometheus/tests/__init__.py create mode 100644 exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py diff --git a/exporter/opentelemetry-exporter-prometheus/LICENSE b/exporter/opentelemetry-exporter-prometheus/LICENSE new file mode 100644 index 00000000000..261eeb9e9f8 --- /dev/null +++ b/exporter/opentelemetry-exporter-prometheus/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/exporter/opentelemetry-exporter-prometheus/MANIFEST.in b/exporter/opentelemetry-exporter-prometheus/MANIFEST.in new file mode 100644 index 00000000000..aed3e33273b --- /dev/null +++ b/exporter/opentelemetry-exporter-prometheus/MANIFEST.in @@ -0,0 +1,9 @@ +graft src +graft tests +global-exclude *.pyc +global-exclude *.pyo +global-exclude __pycache__/* +include CHANGELOG.md +include MANIFEST.in +include README.rst +include LICENSE diff --git a/exporter/opentelemetry-exporter-prometheus/README.rst b/exporter/opentelemetry-exporter-prometheus/README.rst new file mode 100644 index 00000000000..a3eb9200005 --- /dev/null +++ b/exporter/opentelemetry-exporter-prometheus/README.rst @@ -0,0 +1,23 @@ +OpenTelemetry Prometheus Exporter +================================= + +|pypi| + +.. |pypi| image:: https://badge.fury.io/py/opentelemetry-exporter-prometheus.svg + :target: https://pypi.org/project/opentelemetry-exporter-prometheus/ + +This library allows to export metrics data to `Prometheus `_. + +Installation +------------ + +:: + + pip install opentelemetry-exporter-prometheus + +References +---------- + +* `OpenTelemetry Prometheus Exporter `_ +* `Prometheus `_ +* `OpenTelemetry Project `_ diff --git a/exporter/opentelemetry-exporter-prometheus/setup.cfg b/exporter/opentelemetry-exporter-prometheus/setup.cfg new file mode 100644 index 00000000000..b8625698359 --- /dev/null +++ b/exporter/opentelemetry-exporter-prometheus/setup.cfg @@ -0,0 +1,55 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +[metadata] +name = opentelemetry-exporter-prometheus +description = Prometheus Metric Exporter for OpenTelemetry +long_description = file: README.rst +long_description_content_type = text/x-rst +author = OpenTelemetry Authors +author_email = cncf-opentelemetry-contributors@lists.cncf.io +url = https://github.com/open-telemetry/opentelemetry-python/tree/main/exporter/opentelemetry-exporter-prometheus +platforms = any +license = Apache-2.0 +classifiers = + Development Status :: 4 - Beta + Intended Audience :: Developers + License :: OSI Approved :: Apache Software License + Programming Language :: Python + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.5 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + +[options] +python_requires = >=3.5 +package_dir= + =src +packages=find_namespace: +install_requires = + prometheus_client >= 0.5.0, < 1.0.0 + opentelemetry-api == 1.0.0 + opentelemetry-sdk == 1.0.0 + +[options.packages.find] +where = src + +[options.extras_require] +test = + +[options.entry_points] +opentelemetry_exporter = + prometheus = opentelemetry.exporter.prometheus:PrometheusMetricsExporter \ No newline at end of file diff --git a/exporter/opentelemetry-exporter-prometheus/setup.py b/exporter/opentelemetry-exporter-prometheus/setup.py new file mode 100644 index 00000000000..86067a2bd56 --- /dev/null +++ b/exporter/opentelemetry-exporter-prometheus/setup.py @@ -0,0 +1,26 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +import setuptools + +BASE_DIR = os.path.dirname(__file__) +VERSION_FILENAME = os.path.join( + BASE_DIR, "src", "opentelemetry", "exporter", "prometheus", "version.py" +) +PACKAGE_INFO = {} +with open(VERSION_FILENAME) as f: + exec(f.read(), PACKAGE_INFO) + +setuptools.setup(version=PACKAGE_INFO["__version__"]) diff --git a/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py b/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py new file mode 100644 index 00000000000..760c54d3c80 --- /dev/null +++ b/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py @@ -0,0 +1,193 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This library allows export of metrics data to `Prometheus `_. + +Usage +----- + +The **OpenTelemetry Prometheus Exporter** allows export of `OpenTelemetry`_ metrics to `Prometheus`_. + + +.. _Prometheus: https://prometheus.io/ +.. _OpenTelemetry: https://github.com/open-telemetry/opentelemetry-python/ + +.. code:: python + + from opentelemetry import metrics + from opentelemetry.exporter.prometheus import PrometheusMetricsExporter + from opentelemetry.sdk.metrics import Meter + from prometheus_client import start_http_server + + # Start Prometheus client + start_http_server(port=8000, addr="localhost") + + # Meter is responsible for creating and recording metrics + metrics.set_meter_provider(MeterProvider()) + meter = metrics.get_meter(__name__) + # exporter to export metrics to Prometheus + prefix = "MyAppPrefix" + exporter = PrometheusMetricsExporter(prefix) + # Starts the collect/export pipeline for metrics + metrics.get_meter_provider().start_pipeline(meter, exporter, 5) + + counter = meter.create_counter( + "requests", + "number of requests", + "requests", + int, + ) + + # Labels are used to identify key-values that are associated with a specific + # metric that you want to record. These are useful for pre-aggregation and can + # be used to store custom dimensions pertaining to a metric + labels = {"environment": "staging"} + + counter.add(25, labels) + input("Press any key to exit...") + +API +--- +""" + +import collections +import logging +import re +from typing import Iterable, Optional, Sequence, Union + +from prometheus_client.core import ( + REGISTRY, + CounterMetricFamily, + SummaryMetricFamily, + UnknownMetricFamily, +) + +from opentelemetry.metrics import Counter, ValueRecorder +from opentelemetry.sdk.metrics.export import ( + ExportRecord, + MetricsExporter, + MetricsExportResult, +) +from opentelemetry.sdk.metrics.export.aggregate import MinMaxSumCountAggregator + +logger = logging.getLogger(__name__) + + +class PrometheusMetricsExporter(MetricsExporter): + """Prometheus metric exporter for OpenTelemetry. + + Args: + prefix: single-word application prefix relevant to the domain + the metric belongs to. + """ + + def __init__(self, prefix: str = ""): + self._collector = CustomCollector(prefix) + REGISTRY.register(self._collector) + + def export( + self, export_records: Sequence[ExportRecord] + ) -> MetricsExportResult: + self._collector.add_metrics_data(export_records) + return MetricsExportResult.SUCCESS + + def shutdown(self) -> None: + REGISTRY.unregister(self._collector) + + +class CustomCollector: + """CustomCollector represents the Prometheus Collector object + https://github.com/prometheus/client_python#custom-collectors + """ + + def __init__(self, prefix: str = ""): + self._prefix = prefix + self._metrics_to_export = collections.deque() + self._non_letters_nor_digits_re = re.compile( + r"[^\w]", re.UNICODE | re.IGNORECASE + ) + + def add_metrics_data(self, export_records: Sequence[ExportRecord]) -> None: + self._metrics_to_export.append(export_records) + + def collect(self): + """Collect fetches the metrics from OpenTelemetry + and delivers them as Prometheus Metrics. + Collect is invoked every time a prometheus.Gatherer is run + for example when the HTTP endpoint is invoked by Prometheus. + """ + + while self._metrics_to_export: + for export_record in self._metrics_to_export.popleft(): + prometheus_metric = self._translate_to_prometheus( + export_record + ) + if prometheus_metric is not None: + yield prometheus_metric + + def _translate_to_prometheus(self, export_record: ExportRecord): + prometheus_metric = None + label_values = [] + label_keys = [] + for label_tuple in export_record.labels: + label_keys.append(self._sanitize(label_tuple[0])) + label_values.append(label_tuple[1]) + + metric_name = "" + if self._prefix != "": + metric_name = self._prefix + "_" + metric_name += self._sanitize(export_record.instrument.name) + + description = getattr(export_record.instrument, "description", "") + if isinstance(export_record.instrument, Counter): + prometheus_metric = CounterMetricFamily( + name=metric_name, documentation=description, labels=label_keys + ) + prometheus_metric.add_metric( + labels=label_values, value=export_record.aggregator.checkpoint + ) + # TODO: Add support for histograms when supported in OT + elif isinstance(export_record.instrument, ValueRecorder): + value = export_record.aggregator.checkpoint + if isinstance(export_record.aggregator, MinMaxSumCountAggregator): + prometheus_metric = SummaryMetricFamily( + name=metric_name, + documentation=description, + labels=label_keys, + ) + prometheus_metric.add_metric( + labels=label_values, + count_value=value.count, + sum_value=value.sum, + ) + else: + prometheus_metric = UnknownMetricFamily( + name=metric_name, + documentation=description, + labels=label_keys, + ) + prometheus_metric.add_metric(labels=label_values, value=value) + + else: + logger.warning( + "Unsupported metric type. %s", type(export_record.instrument) + ) + return prometheus_metric + + def _sanitize(self, key: str) -> str: + """sanitize the given metric name or label according to Prometheus rule. + Replace all characters other than [A-Za-z0-9_] with '_'. + """ + return self._non_letters_nor_digits_re.sub("_", key) diff --git a/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/version.py b/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/version.py new file mode 100644 index 00000000000..c7a4430aebe --- /dev/null +++ b/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/version.py @@ -0,0 +1,15 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__version__ = "0.19b0" diff --git a/exporter/opentelemetry-exporter-prometheus/tests/__init__.py b/exporter/opentelemetry-exporter-prometheus/tests/__init__.py new file mode 100644 index 00000000000..b0a6f428417 --- /dev/null +++ b/exporter/opentelemetry-exporter-prometheus/tests/__init__.py @@ -0,0 +1,13 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py b/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py new file mode 100644 index 00000000000..5813fba33ca --- /dev/null +++ b/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py @@ -0,0 +1,169 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from unittest import mock + +from prometheus_client import generate_latest +from prometheus_client.core import CounterMetricFamily + +from opentelemetry.exporter.prometheus import ( + CustomCollector, + PrometheusMetricsExporter, +) +from opentelemetry.metrics import get_meter_provider, set_meter_provider +from opentelemetry.sdk import metrics +from opentelemetry.sdk.metrics.export import ExportRecord, MetricsExportResult +from opentelemetry.sdk.metrics.export.aggregate import ( + MinMaxSumCountAggregator, + SumAggregator, +) +from opentelemetry.sdk.util import get_dict_as_key + + +class TestPrometheusMetricExporter(unittest.TestCase): + def setUp(self): + set_meter_provider(metrics.MeterProvider()) + self._meter = get_meter_provider().get_meter(__name__) + self._test_metric = self._meter.create_counter( + "testname", "testdesc", "unit", int, + ) + labels = {"environment": "staging"} + self._labels_key = get_dict_as_key(labels) + + self._mock_registry_register = mock.Mock() + self._registry_register_patch = mock.patch( + "prometheus_client.core.REGISTRY.register", + side_effect=self._mock_registry_register, + ) + + # pylint: disable=protected-access + def test_constructor(self): + """Test the constructor.""" + with self._registry_register_patch: + exporter = PrometheusMetricsExporter("testprefix") + self.assertEqual(exporter._collector._prefix, "testprefix") + self.assertTrue(self._mock_registry_register.called) + + def test_shutdown(self): + with mock.patch( + "prometheus_client.core.REGISTRY.unregister" + ) as registry_unregister_patch: + exporter = PrometheusMetricsExporter() + exporter.shutdown() + self.assertTrue(registry_unregister_patch.called) + + def test_export(self): + with self._registry_register_patch: + record = ExportRecord( + self._test_metric, + self._labels_key, + SumAggregator(), + get_meter_provider().resource, + ) + exporter = PrometheusMetricsExporter() + result = exporter.export([record]) + # pylint: disable=protected-access + self.assertEqual(len(exporter._collector._metrics_to_export), 1) + self.assertIs(result, MetricsExportResult.SUCCESS) + + def test_min_max_sum_aggregator_to_prometheus(self): + meter = get_meter_provider().get_meter(__name__) + metric = meter.create_valuerecorder( + "test@name", "testdesc", "unit", int, [] + ) + labels = {} + key_labels = get_dict_as_key(labels) + aggregator = MinMaxSumCountAggregator() + aggregator.update(123) + aggregator.update(456) + aggregator.take_checkpoint() + record = ExportRecord( + metric, key_labels, aggregator, get_meter_provider().resource + ) + collector = CustomCollector("testprefix") + collector.add_metrics_data([record]) + result_bytes = generate_latest(collector) + result = result_bytes.decode("utf-8") + self.assertIn("testprefix_test_name_count 2.0", result) + self.assertIn("testprefix_test_name_sum 579.0", result) + + def test_counter_to_prometheus(self): + meter = get_meter_provider().get_meter(__name__) + metric = meter.create_counter("test@name", "testdesc", "unit", int,) + labels = {"environment@": "staging", "os": "Windows"} + key_labels = get_dict_as_key(labels) + aggregator = SumAggregator() + aggregator.update(123) + aggregator.take_checkpoint() + record = ExportRecord( + metric, key_labels, aggregator, get_meter_provider().resource + ) + collector = CustomCollector("testprefix") + collector.add_metrics_data([record]) + + for prometheus_metric in collector.collect(): + self.assertEqual(type(prometheus_metric), CounterMetricFamily) + self.assertEqual(prometheus_metric.name, "testprefix_test_name") + self.assertEqual(prometheus_metric.documentation, "testdesc") + self.assertTrue(len(prometheus_metric.samples) == 1) + self.assertEqual(prometheus_metric.samples[0].value, 123) + self.assertTrue(len(prometheus_metric.samples[0].labels) == 2) + self.assertEqual( + prometheus_metric.samples[0].labels["environment_"], "staging" + ) + self.assertEqual( + prometheus_metric.samples[0].labels["os"], "Windows" + ) + + # TODO: Add unit test once GaugeAggregator is available + # TODO: Add unit test once Measure Aggregators are available + + def test_invalid_metric(self): + meter = get_meter_provider().get_meter(__name__) + metric = StubMetric("tesname", "testdesc", "unit", int, meter) + labels = {"environment": "staging"} + key_labels = get_dict_as_key(labels) + record = ExportRecord( + metric, key_labels, None, get_meter_provider().resource + ) + collector = CustomCollector("testprefix") + collector.add_metrics_data([record]) + collector.collect() + self.assertLogs("opentelemetry.exporter.prometheus", level="WARNING") + + def test_sanitize(self): + collector = CustomCollector("testprefix") + self.assertEqual( + collector._sanitize("1!2@3#4$5%6^7&8*9(0)_-"), + "1_2_3_4_5_6_7_8_9_0___", + ) + self.assertEqual(collector._sanitize(",./?;:[]{}"), "__________") + self.assertEqual(collector._sanitize("TestString"), "TestString") + self.assertEqual(collector._sanitize("aAbBcC_12_oi"), "aAbBcC_12_oi") + + +class StubMetric(metrics.Metric): + def __init__( + self, + name: str, + description: str, + unit: str, + value_type, + meter, + enabled: bool = True, + ): + super().__init__( + name, description, unit, value_type, meter, enabled=enabled, + ) From 9698be2215241b4c0de4fbdcf02a960851c9c1f6 Mon Sep 17 00:00:00 2001 From: Alex Boten Date: Mon, 13 Dec 2021 16:20:29 -0800 Subject: [PATCH 02/31] update references --- exporter/opentelemetry-exporter-prometheus/setup.cfg | 4 ++-- .../src/opentelemetry/exporter/prometheus/__init__.py | 10 +++++----- tox.ini | 6 ++++++ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/exporter/opentelemetry-exporter-prometheus/setup.cfg b/exporter/opentelemetry-exporter-prometheus/setup.cfg index b8625698359..f6e04dc12ad 100644 --- a/exporter/opentelemetry-exporter-prometheus/setup.cfg +++ b/exporter/opentelemetry-exporter-prometheus/setup.cfg @@ -41,8 +41,8 @@ package_dir= packages=find_namespace: install_requires = prometheus_client >= 0.5.0, < 1.0.0 - opentelemetry-api == 1.0.0 - opentelemetry-sdk == 1.0.0 + opentelemetry-api >= 1.7.1 + opentelemetry-sdk >= 1.7.1 [options.packages.find] where = src diff --git a/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py b/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py index 760c54d3c80..e51af6f0391 100644 --- a/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py +++ b/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py @@ -74,10 +74,10 @@ UnknownMetricFamily, ) -from opentelemetry.metrics import Counter, ValueRecorder -from opentelemetry.sdk.metrics.export import ( +from opentelemetry._metrics import Counter, Histogram +from opentelemetry.sdk._metrics.export import ( ExportRecord, - MetricsExporter, + MetricExporter, MetricsExportResult, ) from opentelemetry.sdk.metrics.export.aggregate import MinMaxSumCountAggregator @@ -85,7 +85,7 @@ logger = logging.getLogger(__name__) -class PrometheusMetricsExporter(MetricsExporter): +class PrometheusMetricExporter(MetricExporter): """Prometheus metric exporter for OpenTelemetry. Args: @@ -159,7 +159,7 @@ def _translate_to_prometheus(self, export_record: ExportRecord): labels=label_values, value=export_record.aggregator.checkpoint ) # TODO: Add support for histograms when supported in OT - elif isinstance(export_record.instrument, ValueRecorder): + elif isinstance(export_record.instrument, Histogram): value = export_record.aggregator.checkpoint if isinstance(export_record.aggregator, MinMaxSumCountAggregator): prometheus_metric = SummaryMetricFamily( diff --git a/tox.ini b/tox.ini index 694d3d78a64..73e5a796cc5 100644 --- a/tox.ini +++ b/tox.ini @@ -42,6 +42,9 @@ envlist = py3{6,7,8,9,10}-opentelemetry-exporter-otlp-proto-http pypy3-opentelemetry-exporter-otlp-proto-http + py3{6,7,8,9,10}-opentelemetry-exporter-prometheus + pypy3-opentelemetry-exporter-prometheus + ; opentelemetry-exporter-zipkin py3{6,7,8,9,10}-opentelemetry-exporter-zipkin-combined pypy3-opentelemetry-exporter-zipkin-combined @@ -97,6 +100,7 @@ changedir = exporter-otlp-combined: exporter/opentelemetry-exporter-otlp/tests exporter-otlp-proto-grpc: exporter/opentelemetry-exporter-otlp-proto-grpc/tests exporter-otlp-proto-http: exporter/opentelemetry-exporter-otlp-proto-http/tests + exporter-prometheus: exporter/opentelemetry-exporter-prometheus/tests exporter-zipkin-combined: exporter/opentelemetry-exporter-zipkin/tests exporter-zipkin-proto-http: exporter/opentelemetry-exporter-zipkin-proto-http/tests exporter-zipkin-json: exporter/opentelemetry-exporter-zipkin-json/tests @@ -140,6 +144,8 @@ commands_pre = opentracing-shim: pip install {toxinidir}/opentelemetry-sdk opentracing-shim: pip install {toxinidir}/shim/opentelemetry-opentracing-shim + exporter-prometheus: pip install {toxinidir}/exporter/opentelemetry-exporter-prometheus + exporter-zipkin-combined: pip install {toxinidir}/exporter/opentelemetry-exporter-zipkin-json exporter-zipkin-combined: pip install {toxinidir}/exporter/opentelemetry-exporter-zipkin-proto-http exporter-zipkin-combined: pip install {toxinidir}/exporter/opentelemetry-exporter-zipkin From 982480f825840677a01a2849acf0225b12ce5485 Mon Sep 17 00:00:00 2001 From: Alex Boten Date: Wed, 23 Feb 2022 14:19:12 -0800 Subject: [PATCH 03/31] update more references --- .../setup.cfg | 2 +- .../exporter/prometheus/__init__.py | 84 +++++----- .../tests/test_prometheus_exporter.py | 154 +++++++++--------- 3 files changed, 123 insertions(+), 117 deletions(-) diff --git a/exporter/opentelemetry-exporter-prometheus/setup.cfg b/exporter/opentelemetry-exporter-prometheus/setup.cfg index f6e04dc12ad..5227e1a00fe 100644 --- a/exporter/opentelemetry-exporter-prometheus/setup.cfg +++ b/exporter/opentelemetry-exporter-prometheus/setup.cfg @@ -52,4 +52,4 @@ test = [options.entry_points] opentelemetry_exporter = - prometheus = opentelemetry.exporter.prometheus:PrometheusMetricsExporter \ No newline at end of file + prometheus = opentelemetry.exporter.prometheus:PrometheusMetricExporter \ No newline at end of file diff --git a/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py b/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py index e51af6f0391..cecb577b1b1 100644 --- a/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py +++ b/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py @@ -27,7 +27,7 @@ .. code:: python from opentelemetry import metrics - from opentelemetry.exporter.prometheus import PrometheusMetricsExporter + from opentelemetry.exporter.prometheus import PrometheusMetricExporter from opentelemetry.sdk.metrics import Meter from prometheus_client import start_http_server @@ -39,7 +39,7 @@ meter = metrics.get_meter(__name__) # exporter to export metrics to Prometheus prefix = "MyAppPrefix" - exporter = PrometheusMetricsExporter(prefix) + exporter = PrometheusMetricExporter(prefix) # Starts the collect/export pipeline for metrics metrics.get_meter_provider().start_pipeline(meter, exporter, 5) @@ -70,17 +70,21 @@ from prometheus_client.core import ( REGISTRY, CounterMetricFamily, - SummaryMetricFamily, - UnknownMetricFamily, + GaugeMetricFamily, + HistogramMetricFamily, ) -from opentelemetry._metrics import Counter, Histogram from opentelemetry.sdk._metrics.export import ( - ExportRecord, MetricExporter, - MetricsExportResult, + MetricExportResult, +) +from opentelemetry.sdk._metrics.point import ( + AggregationTemporality, + Gauge, + Histogram, + Metric, + Sum, ) -from opentelemetry.sdk.metrics.export.aggregate import MinMaxSumCountAggregator logger = logging.getLogger(__name__) @@ -97,11 +101,9 @@ def __init__(self, prefix: str = ""): self._collector = CustomCollector(prefix) REGISTRY.register(self._collector) - def export( - self, export_records: Sequence[ExportRecord] - ) -> MetricsExportResult: + def export(self, export_records: Sequence[Metric]) -> MetricExportResult: self._collector.add_metrics_data(export_records) - return MetricsExportResult.SUCCESS + return MetricExportResult.SUCCESS def shutdown(self) -> None: REGISTRY.unregister(self._collector) @@ -119,7 +121,7 @@ def __init__(self, prefix: str = ""): r"[^\w]", re.UNICODE | re.IGNORECASE ) - def add_metrics_data(self, export_records: Sequence[ExportRecord]) -> None: + def add_metrics_data(self, export_records: Sequence[Metric]) -> None: self._metrics_to_export.append(export_records) def collect(self): @@ -137,52 +139,48 @@ def collect(self): if prometheus_metric is not None: yield prometheus_metric - def _translate_to_prometheus(self, export_record: ExportRecord): + def _translate_to_prometheus(self, export_record: Metric): prometheus_metric = None label_values = [] label_keys = [] - for label_tuple in export_record.labels: - label_keys.append(self._sanitize(label_tuple[0])) - label_values.append(label_tuple[1]) + for key, value in export_record.attributes.items(): + label_keys.append(self._sanitize(key)) + label_values.append(str(value)) metric_name = "" if self._prefix != "": metric_name = self._prefix + "_" - metric_name += self._sanitize(export_record.instrument.name) + metric_name += self._sanitize(export_record.name) - description = getattr(export_record.instrument, "description", "") - if isinstance(export_record.instrument, Counter): + description = export_record.description or "" + if isinstance(export_record.point, Sum): prometheus_metric = CounterMetricFamily( name=metric_name, documentation=description, labels=label_keys ) prometheus_metric.add_metric( - labels=label_values, value=export_record.aggregator.checkpoint + labels=label_values, value=export_record.point.value + ) + elif isinstance(export_record.point, Gauge): + prometheus_metric = GaugeMetricFamily( + name=metric_name, documentation=description, labels=label_keys + ) + prometheus_metric.add_metric( + labels=label_values, value=export_record.point.value ) # TODO: Add support for histograms when supported in OT - elif isinstance(export_record.instrument, Histogram): - value = export_record.aggregator.checkpoint - if isinstance(export_record.aggregator, MinMaxSumCountAggregator): - prometheus_metric = SummaryMetricFamily( - name=metric_name, - documentation=description, - labels=label_keys, - ) - prometheus_metric.add_metric( - labels=label_values, - count_value=value.count, - sum_value=value.sum, - ) - else: - prometheus_metric = UnknownMetricFamily( - name=metric_name, - documentation=description, - labels=label_keys, - ) - prometheus_metric.add_metric(labels=label_values, value=value) - + # elif isinstance(export_record.point, Histogram): + # value = export_record.point.sum + # prometheus_metric = HistogramMetricFamily( + # name=metric_name, + # documentation=description, + # labels=label_keys, + # ) + # prometheus_metric.add_metric(labels=label_values, buckets=export_record.point.explicit_bounds, sum_value=value) + # TODO: add support for Summary once implemented + # elif isinstance(export_record.point, Summary): else: logger.warning( - "Unsupported metric type. %s", type(export_record.instrument) + "Unsupported metric type. %s", type(export_record.point) ) return prometheus_metric diff --git a/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py b/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py index 5813fba33ca..8834cd9be7a 100644 --- a/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py +++ b/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py @@ -16,28 +16,37 @@ from unittest import mock from prometheus_client import generate_latest -from prometheus_client.core import CounterMetricFamily +from prometheus_client.core import CounterMetricFamily, GaugeMetricFamily +from opentelemetry._metrics import get_meter_provider, set_meter_provider from opentelemetry.exporter.prometheus import ( CustomCollector, - PrometheusMetricsExporter, + PrometheusMetricExporter, ) -from opentelemetry.metrics import get_meter_provider, set_meter_provider -from opentelemetry.sdk import metrics -from opentelemetry.sdk.metrics.export import ExportRecord, MetricsExportResult -from opentelemetry.sdk.metrics.export.aggregate import ( - MinMaxSumCountAggregator, - SumAggregator, +from opentelemetry.sdk._metrics import MeterProvider +from opentelemetry.sdk._metrics.export import MetricExportResult +from opentelemetry.sdk._metrics.point import ( + AggregationTemporality, + Histogram, + Metric, ) from opentelemetry.sdk.util import get_dict_as_key +from opentelemetry.test.metrictestutil import ( + _generate_gauge, + _generate_metric, + _generate_sum, + _generate_unsupported_metric, +) class TestPrometheusMetricExporter(unittest.TestCase): def setUp(self): - set_meter_provider(metrics.MeterProvider()) + set_meter_provider(MeterProvider()) self._meter = get_meter_provider().get_meter(__name__) self._test_metric = self._meter.create_counter( - "testname", "testdesc", "unit", int, + "testname", + description="testdesc", + unit="unit", ) labels = {"environment": "staging"} self._labels_key = get_dict_as_key(labels) @@ -52,7 +61,7 @@ def setUp(self): def test_constructor(self): """Test the constructor.""" with self._registry_register_patch: - exporter = PrometheusMetricsExporter("testprefix") + exporter = PrometheusMetricExporter("testprefix") self.assertEqual(exporter._collector._prefix, "testprefix") self.assertTrue(self._mock_registry_register.called) @@ -60,62 +69,55 @@ def test_shutdown(self): with mock.patch( "prometheus_client.core.REGISTRY.unregister" ) as registry_unregister_patch: - exporter = PrometheusMetricsExporter() + exporter = PrometheusMetricExporter() exporter.shutdown() self.assertTrue(registry_unregister_patch.called) def test_export(self): with self._registry_register_patch: - record = ExportRecord( - self._test_metric, - self._labels_key, - SumAggregator(), - get_meter_provider().resource, - ) - exporter = PrometheusMetricsExporter() + record = _generate_sum("sum_int", 33) + exporter = PrometheusMetricExporter() result = exporter.export([record]) # pylint: disable=protected-access self.assertEqual(len(exporter._collector._metrics_to_export), 1) - self.assertIs(result, MetricsExportResult.SUCCESS) - - def test_min_max_sum_aggregator_to_prometheus(self): - meter = get_meter_provider().get_meter(__name__) - metric = meter.create_valuerecorder( - "test@name", "testdesc", "unit", int, [] + self.assertIs(result, MetricExportResult.SUCCESS) + + # # TODO: Add unit test for histogram + def test_histogram_to_prometheus(self): + record = _generate_metric( + "test@name", + Histogram( + time_unix_nano=1641946016139533244, + start_time_unix_nano=1641946016139533244, + bucket_counts=[1, 1], + sum=579.0, + explicit_bounds=[123.0, 456.0], + aggregation_temporality=AggregationTemporality.DELTA, + ), ) - labels = {} - key_labels = get_dict_as_key(labels) - aggregator = MinMaxSumCountAggregator() - aggregator.update(123) - aggregator.update(456) - aggregator.take_checkpoint() - record = ExportRecord( - metric, key_labels, aggregator, get_meter_provider().resource - ) - collector = CustomCollector("testprefix") - collector.add_metrics_data([record]) - result_bytes = generate_latest(collector) - result = result_bytes.decode("utf-8") - self.assertIn("testprefix_test_name_count 2.0", result) - self.assertIn("testprefix_test_name_sum 579.0", result) - - def test_counter_to_prometheus(self): - meter = get_meter_provider().get_meter(__name__) - metric = meter.create_counter("test@name", "testdesc", "unit", int,) + + # collector = CustomCollector("testprefix") + # collector.add_metrics_data([record]) + # result_bytes = generate_latest(collector) + # result = result_bytes.decode("utf-8") + # self.assertIn("testprefix_test_name_count 2.0", result) + # self.assertIn("testprefix_test_name_sum 579.0", result) + + def test_sum_to_prometheus(self): labels = {"environment@": "staging", "os": "Windows"} - key_labels = get_dict_as_key(labels) - aggregator = SumAggregator() - aggregator.update(123) - aggregator.take_checkpoint() - record = ExportRecord( - metric, key_labels, aggregator, get_meter_provider().resource + record = _generate_sum( + "test@sum", + 123, + attributes=labels, + description="testdesc", + unit="testunit", ) collector = CustomCollector("testprefix") collector.add_metrics_data([record]) for prometheus_metric in collector.collect(): self.assertEqual(type(prometheus_metric), CounterMetricFamily) - self.assertEqual(prometheus_metric.name, "testprefix_test_name") + self.assertEqual(prometheus_metric.name, "testprefix_test_sum") self.assertEqual(prometheus_metric.documentation, "testdesc") self.assertTrue(len(prometheus_metric.samples) == 1) self.assertEqual(prometheus_metric.samples[0].value, 123) @@ -127,16 +129,37 @@ def test_counter_to_prometheus(self): prometheus_metric.samples[0].labels["os"], "Windows" ) - # TODO: Add unit test once GaugeAggregator is available - # TODO: Add unit test once Measure Aggregators are available + def test_gauge_to_prometheus(self): + labels = {"environment@": "dev", "os": "Unix"} + record = _generate_gauge( + "test@gauge", + 123, + attributes=labels, + description="testdesc", + unit="testunit", + ) + collector = CustomCollector("testprefix") + collector.add_metrics_data([record]) + + for prometheus_metric in collector.collect(): + self.assertEqual(type(prometheus_metric), GaugeMetricFamily) + self.assertEqual(prometheus_metric.name, "testprefix_test_gauge") + self.assertEqual(prometheus_metric.documentation, "testdesc") + self.assertTrue(len(prometheus_metric.samples) == 1) + self.assertEqual(prometheus_metric.samples[0].value, 123) + self.assertTrue(len(prometheus_metric.samples[0].labels) == 2) + self.assertEqual( + prometheus_metric.samples[0].labels["environment_"], "dev" + ) + self.assertEqual(prometheus_metric.samples[0].labels["os"], "Unix") def test_invalid_metric(self): - meter = get_meter_provider().get_meter(__name__) - metric = StubMetric("tesname", "testdesc", "unit", int, meter) labels = {"environment": "staging"} - key_labels = get_dict_as_key(labels) - record = ExportRecord( - metric, key_labels, None, get_meter_provider().resource + record = _generate_unsupported_metric( + "tesname", + attributes=labels, + description="testdesc", + unit="testunit", ) collector = CustomCollector("testprefix") collector.add_metrics_data([record]) @@ -152,18 +175,3 @@ def test_sanitize(self): self.assertEqual(collector._sanitize(",./?;:[]{}"), "__________") self.assertEqual(collector._sanitize("TestString"), "TestString") self.assertEqual(collector._sanitize("aAbBcC_12_oi"), "aAbBcC_12_oi") - - -class StubMetric(metrics.Metric): - def __init__( - self, - name: str, - description: str, - unit: str, - value_type, - meter, - enabled: bool = True, - ): - super().__init__( - name, description, unit, value_type, meter, enabled=enabled, - ) From c14d9762ba38b8074a18063521feebde498747bf Mon Sep 17 00:00:00 2001 From: Alex Boten Date: Fri, 25 Feb 2022 09:17:53 -0800 Subject: [PATCH 04/31] add histogram --- .../exporter/prometheus/__init__.py | 95 +++++++++++-------- .../tests/test_prometheus_exporter.py | 38 +++----- 2 files changed, 67 insertions(+), 66 deletions(-) diff --git a/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py b/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py index cecb577b1b1..0262be4a863 100644 --- a/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py +++ b/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py @@ -65,26 +65,15 @@ import collections import logging import re -from typing import Iterable, Optional, Sequence, Union +from typing import Optional, Sequence, Tuple -from prometheus_client.core import ( - REGISTRY, - CounterMetricFamily, - GaugeMetricFamily, - HistogramMetricFamily, -) +from prometheus_client import core from opentelemetry.sdk._metrics.export import ( MetricExporter, MetricExportResult, ) -from opentelemetry.sdk._metrics.point import ( - AggregationTemporality, - Gauge, - Histogram, - Metric, - Sum, -) +from opentelemetry.sdk._metrics.point import Gauge, Histogram, Metric, Sum logger = logging.getLogger(__name__) @@ -99,14 +88,14 @@ class PrometheusMetricExporter(MetricExporter): def __init__(self, prefix: str = ""): self._collector = CustomCollector(prefix) - REGISTRY.register(self._collector) + core.REGISTRY.register(self._collector) def export(self, export_records: Sequence[Metric]) -> MetricExportResult: self._collector.add_metrics_data(export_records) return MetricExportResult.SUCCESS def shutdown(self) -> None: - REGISTRY.unregister(self._collector) + core.REGISTRY.unregister(self._collector) class CustomCollector: @@ -124,7 +113,7 @@ def __init__(self, prefix: str = ""): def add_metrics_data(self, export_records: Sequence[Metric]) -> None: self._metrics_to_export.append(export_records) - def collect(self): + def collect(self) -> None: """Collect fetches the metrics from OpenTelemetry and delivers them as Prometheus Metrics. Collect is invoked every time a prometheus.Gatherer is run @@ -139,49 +128,71 @@ def collect(self): if prometheus_metric is not None: yield prometheus_metric - def _translate_to_prometheus(self, export_record: Metric): + def _convert_buckets(self, metric: Metric) -> Sequence[Tuple[str, int]]: + buckets = [] + total_count = 0 + for i in range(0, len(metric.point.bucket_counts)): + total_count += metric.point.bucket_counts[i] + buckets.append( + ( + f"{metric.point.explicit_bounds[i]}", + total_count, + ) + ) + return buckets + + def _translate_to_prometheus( + self, metric: Metric + ) -> Optional[core.Metric]: prometheus_metric = None label_values = [] label_keys = [] - for key, value in export_record.attributes.items(): + for key, value in metric.attributes.items(): label_keys.append(self._sanitize(key)) label_values.append(str(value)) metric_name = "" if self._prefix != "": metric_name = self._prefix + "_" - metric_name += self._sanitize(export_record.name) - - description = export_record.description or "" - if isinstance(export_record.point, Sum): - prometheus_metric = CounterMetricFamily( - name=metric_name, documentation=description, labels=label_keys + metric_name += self._sanitize(metric.name) + + description = metric.description or "" + if isinstance(metric.point, Sum): + prometheus_metric = core.CounterMetricFamily( + name=metric_name, + documentation=description, + labels=label_keys, + unit=metric.unit, ) prometheus_metric.add_metric( - labels=label_values, value=export_record.point.value + labels=label_values, value=metric.point.value ) - elif isinstance(export_record.point, Gauge): - prometheus_metric = GaugeMetricFamily( - name=metric_name, documentation=description, labels=label_keys + elif isinstance(metric.point, Gauge): + prometheus_metric = core.GaugeMetricFamily( + name=metric_name, + documentation=description, + labels=label_keys, + unit=metric.unit, ) prometheus_metric.add_metric( - labels=label_values, value=export_record.point.value + labels=label_values, value=metric.point.value + ) + elif isinstance(metric.point, Histogram): + value = metric.point.sum + prometheus_metric = core.HistogramMetricFamily( + name=metric_name, + documentation=description, + labels=label_keys, + unit=metric.unit, + ) + buckets = self._convert_buckets(metric) + prometheus_metric.add_metric( + labels=label_values, buckets=buckets, sum_value=value ) - # TODO: Add support for histograms when supported in OT - # elif isinstance(export_record.point, Histogram): - # value = export_record.point.sum - # prometheus_metric = HistogramMetricFamily( - # name=metric_name, - # documentation=description, - # labels=label_keys, - # ) - # prometheus_metric.add_metric(labels=label_values, buckets=export_record.point.explicit_bounds, sum_value=value) # TODO: add support for Summary once implemented # elif isinstance(export_record.point, Summary): else: - logger.warning( - "Unsupported metric type. %s", type(export_record.point) - ) + logger.warning("Unsupported metric type. %s", type(metric.point)) return prometheus_metric def _sanitize(self, key: str) -> str: diff --git a/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py b/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py index 8834cd9be7a..5646da06124 100644 --- a/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py +++ b/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py @@ -25,11 +25,7 @@ ) from opentelemetry.sdk._metrics import MeterProvider from opentelemetry.sdk._metrics.export import MetricExportResult -from opentelemetry.sdk._metrics.point import ( - AggregationTemporality, - Histogram, - Metric, -) +from opentelemetry.sdk._metrics.point import AggregationTemporality, Histogram from opentelemetry.sdk.util import get_dict_as_key from opentelemetry.test.metrictestutil import ( _generate_gauge, @@ -41,16 +37,6 @@ class TestPrometheusMetricExporter(unittest.TestCase): def setUp(self): - set_meter_provider(MeterProvider()) - self._meter = get_meter_provider().get_meter(__name__) - self._test_metric = self._meter.create_counter( - "testname", - description="testdesc", - unit="unit", - ) - labels = {"environment": "staging"} - self._labels_key = get_dict_as_key(labels) - self._mock_registry_register = mock.Mock() self._registry_register_patch = mock.patch( "prometheus_client.core.REGISTRY.register", @@ -82,7 +68,6 @@ def test_export(self): self.assertEqual(len(exporter._collector._metrics_to_export), 1) self.assertIs(result, MetricExportResult.SUCCESS) - # # TODO: Add unit test for histogram def test_histogram_to_prometheus(self): record = _generate_metric( "test@name", @@ -94,14 +79,15 @@ def test_histogram_to_prometheus(self): explicit_bounds=[123.0, 456.0], aggregation_temporality=AggregationTemporality.DELTA, ), + attributes={"histo": 1}, ) - # collector = CustomCollector("testprefix") - # collector.add_metrics_data([record]) - # result_bytes = generate_latest(collector) - # result = result_bytes.decode("utf-8") - # self.assertIn("testprefix_test_name_count 2.0", result) - # self.assertIn("testprefix_test_name_sum 579.0", result) + collector = CustomCollector("testprefix") + collector.add_metrics_data([record]) + result_bytes = generate_latest(collector) + result = result_bytes.decode("utf-8") + self.assertIn('testprefix_test_name_s_sum{histo="1"} 579.0', result) + self.assertIn('testprefix_test_name_s_count{histo="1"} 2.0', result) def test_sum_to_prometheus(self): labels = {"environment@": "staging", "os": "Windows"} @@ -117,7 +103,9 @@ def test_sum_to_prometheus(self): for prometheus_metric in collector.collect(): self.assertEqual(type(prometheus_metric), CounterMetricFamily) - self.assertEqual(prometheus_metric.name, "testprefix_test_sum") + self.assertEqual( + prometheus_metric.name, "testprefix_test_sum_testunit" + ) self.assertEqual(prometheus_metric.documentation, "testdesc") self.assertTrue(len(prometheus_metric.samples) == 1) self.assertEqual(prometheus_metric.samples[0].value, 123) @@ -143,7 +131,9 @@ def test_gauge_to_prometheus(self): for prometheus_metric in collector.collect(): self.assertEqual(type(prometheus_metric), GaugeMetricFamily) - self.assertEqual(prometheus_metric.name, "testprefix_test_gauge") + self.assertEqual( + prometheus_metric.name, "testprefix_test_gauge_testunit" + ) self.assertEqual(prometheus_metric.documentation, "testdesc") self.assertTrue(len(prometheus_metric.samples) == 1) self.assertEqual(prometheus_metric.samples[0].value, 123) From 0baa1aa24fc4a7119e81678494b102498bb6769a Mon Sep 17 00:00:00 2001 From: Alex Boten Date: Fri, 25 Feb 2022 09:19:27 -0800 Subject: [PATCH 05/31] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4031880bf78..9837607d487 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#2461](https://github.com/open-telemetry/opentelemetry-python/pull/2461)) - fix exception handling in get_aggregated_resources ([#2464](https://github.com/open-telemetry/opentelemetry-python/pull/2464)) +- [exporter/opentelemetry-exporter-prometheus] restore package using the new metrics API + ([#2321](https://github.com/open-telemetry/opentelemetry-python/pull/2321)) ## [1.9.1-0.28b1](https://github.com/open-telemetry/opentelemetry-python/releases/tag/v1.9.1-0.28b1) - 2022-01-29 From 4c2d46f4b925f5f90486efdd446b44e688cf42b6 Mon Sep 17 00:00:00 2001 From: Alex Boten Date: Fri, 25 Feb 2022 09:37:38 -0800 Subject: [PATCH 06/31] fix lint --- .../tests/test_prometheus_exporter.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py b/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py index 5646da06124..b10ac67898a 100644 --- a/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py +++ b/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py @@ -18,15 +18,12 @@ from prometheus_client import generate_latest from prometheus_client.core import CounterMetricFamily, GaugeMetricFamily -from opentelemetry._metrics import get_meter_provider, set_meter_provider from opentelemetry.exporter.prometheus import ( CustomCollector, PrometheusMetricExporter, ) -from opentelemetry.sdk._metrics import MeterProvider from opentelemetry.sdk._metrics.export import MetricExportResult from opentelemetry.sdk._metrics.point import AggregationTemporality, Histogram -from opentelemetry.sdk.util import get_dict_as_key from opentelemetry.test.metrictestutil import ( _generate_gauge, _generate_metric, From fdc2c6b8db82b34f4ebc56c4aa0ecd0b1834226c Mon Sep 17 00:00:00 2001 From: Alex Boten Date: Fri, 25 Feb 2022 10:21:57 -0800 Subject: [PATCH 07/31] appease pylint --- .../exporter/prometheus/__init__.py | 39 ++++++++++--------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py b/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py index 0262be4a863..676e2f6736d 100644 --- a/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py +++ b/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py @@ -18,7 +18,8 @@ Usage ----- -The **OpenTelemetry Prometheus Exporter** allows export of `OpenTelemetry`_ metrics to `Prometheus`_. +The **OpenTelemetry Prometheus Exporter** allows export of `OpenTelemetry`_ +metrics to `Prometheus`_. .. _Prometheus: https://prometheus.io/ @@ -78,6 +79,20 @@ logger = logging.getLogger(__name__) + +def _convert_buckets(metric: Metric) -> Sequence[Tuple[str, int]]: + buckets = [] + total_count = 0 + for index, value in enumerate(metric.point.bucket_counts): + total_count += value + buckets.append( + ( + f"{metric.point.explicit_bounds[index]}", + total_count, + ) + ) + return buckets + class PrometheusMetricExporter(MetricExporter): """Prometheus metric exporter for OpenTelemetry. @@ -90,8 +105,8 @@ def __init__(self, prefix: str = ""): self._collector = CustomCollector(prefix) core.REGISTRY.register(self._collector) - def export(self, export_records: Sequence[Metric]) -> MetricExportResult: - self._collector.add_metrics_data(export_records) + def export(self, metrics: Sequence[Metric]) -> MetricExportResult: + self._collector.add_metrics_data(metrics) return MetricExportResult.SUCCESS def shutdown(self) -> None: @@ -111,6 +126,7 @@ def __init__(self, prefix: str = ""): ) def add_metrics_data(self, export_records: Sequence[Metric]) -> None: + """Add metrics to Prometheus data""" self._metrics_to_export.append(export_records) def collect(self) -> None: @@ -128,19 +144,6 @@ def collect(self) -> None: if prometheus_metric is not None: yield prometheus_metric - def _convert_buckets(self, metric: Metric) -> Sequence[Tuple[str, int]]: - buckets = [] - total_count = 0 - for i in range(0, len(metric.point.bucket_counts)): - total_count += metric.point.bucket_counts[i] - buckets.append( - ( - f"{metric.point.explicit_bounds[i]}", - total_count, - ) - ) - return buckets - def _translate_to_prometheus( self, metric: Metric ) -> Optional[core.Metric]: @@ -185,12 +188,10 @@ def _translate_to_prometheus( labels=label_keys, unit=metric.unit, ) - buckets = self._convert_buckets(metric) + buckets = _convert_buckets(metric) prometheus_metric.add_metric( labels=label_values, buckets=buckets, sum_value=value ) - # TODO: add support for Summary once implemented - # elif isinstance(export_record.point, Summary): else: logger.warning("Unsupported metric type. %s", type(metric.point)) return prometheus_metric From 0ef40d51bc63ba5811a75d579d7b26a95a43477b Mon Sep 17 00:00:00 2001 From: Alex Boten Date: Fri, 25 Feb 2022 10:37:52 -0800 Subject: [PATCH 08/31] fix spacing --- .../src/opentelemetry/exporter/prometheus/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py b/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py index 676e2f6736d..cfeeca3c000 100644 --- a/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py +++ b/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py @@ -79,7 +79,6 @@ logger = logging.getLogger(__name__) - def _convert_buckets(metric: Metric) -> Sequence[Tuple[str, int]]: buckets = [] total_count = 0 @@ -93,6 +92,7 @@ def _convert_buckets(metric: Metric) -> Sequence[Tuple[str, int]]: ) return buckets + class PrometheusMetricExporter(MetricExporter): """Prometheus metric exporter for OpenTelemetry. From 384889009d3f1cdbf8e16ac471c77c4ff6ab866c Mon Sep 17 00:00:00 2001 From: Alex Boten Date: Fri, 25 Feb 2022 11:25:06 -0800 Subject: [PATCH 09/31] update version --- exporter/opentelemetry-exporter-prometheus/setup.cfg | 3 +-- .../src/opentelemetry/exporter/prometheus/version.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/exporter/opentelemetry-exporter-prometheus/setup.cfg b/exporter/opentelemetry-exporter-prometheus/setup.cfg index 5227e1a00fe..878dcc80185 100644 --- a/exporter/opentelemetry-exporter-prometheus/setup.cfg +++ b/exporter/opentelemetry-exporter-prometheus/setup.cfg @@ -28,7 +28,6 @@ classifiers = License :: OSI Approved :: Apache Software License Programming Language :: Python Programming Language :: Python :: 3 - Programming Language :: Python :: 3.5 Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 @@ -52,4 +51,4 @@ test = [options.entry_points] opentelemetry_exporter = - prometheus = opentelemetry.exporter.prometheus:PrometheusMetricExporter \ No newline at end of file + prometheus = opentelemetry.exporter.prometheus:PrometheusMetricExporter diff --git a/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/version.py b/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/version.py index c7a4430aebe..a94d4ed4462 100644 --- a/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/version.py +++ b/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "0.19b0" +__version__ = "0.28b1" From 50cd688b9d1fc5735f296b025b114537bcdd6cd7 Mon Sep 17 00:00:00 2001 From: Alex Boten Date: Mon, 28 Feb 2022 08:27:40 -0800 Subject: [PATCH 10/31] ensure prometheus exporter is installed in tox --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 73e5a796cc5..867f1518700 100644 --- a/tox.ini +++ b/tox.ini @@ -208,6 +208,7 @@ commands_pre = python -m pip install -e {toxinidir}/exporter/opentelemetry-exporter-otlp-proto-grpc[test] python -m pip install -e {toxinidir}/exporter/opentelemetry-exporter-otlp-proto-http[test] python -m pip install -e {toxinidir}/exporter/opentelemetry-exporter-otlp[test] + python -m pip install -e {toxinidir}/exporter/opentelemetry-exporter-prometheus[test] python -m pip install -e {toxinidir}/exporter/opentelemetry-exporter-zipkin-json[test] python -m pip install -e {toxinidir}/exporter/opentelemetry-exporter-zipkin-proto-http[test] python -m pip install -e {toxinidir}/exporter/opentelemetry-exporter-zipkin[test] From cbcbcefb760399d2f76bdeb00995df0b0735a521 Mon Sep 17 00:00:00 2001 From: Alex Boten Date: Mon, 28 Feb 2022 09:08:16 -0800 Subject: [PATCH 11/31] Apply suggestions from code review Co-authored-by: Srikanth Chekuri --- exporter/opentelemetry-exporter-prometheus/setup.cfg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/exporter/opentelemetry-exporter-prometheus/setup.cfg b/exporter/opentelemetry-exporter-prometheus/setup.cfg index 878dcc80185..68f953d92d0 100644 --- a/exporter/opentelemetry-exporter-prometheus/setup.cfg +++ b/exporter/opentelemetry-exporter-prometheus/setup.cfg @@ -32,9 +32,10 @@ classifiers = Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 [options] -python_requires = >=3.5 +python_requires = >=3.6 package_dir= =src packages=find_namespace: From 348c2cfaec341bb32c4b4ab0342cdce49cc7d76e Mon Sep 17 00:00:00 2001 From: Alex Boten Date: Mon, 28 Feb 2022 09:20:33 -0800 Subject: [PATCH 12/31] make CustomCollector private --- .../opentelemetry/exporter/prometheus/__init__.py | 6 +++--- .../tests/test_prometheus_exporter.py | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py b/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py index cfeeca3c000..387ac32de44 100644 --- a/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py +++ b/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py @@ -102,7 +102,7 @@ class PrometheusMetricExporter(MetricExporter): """ def __init__(self, prefix: str = ""): - self._collector = CustomCollector(prefix) + self._collector = _CustomCollector(prefix) core.REGISTRY.register(self._collector) def export(self, metrics: Sequence[Metric]) -> MetricExportResult: @@ -113,8 +113,8 @@ def shutdown(self) -> None: core.REGISTRY.unregister(self._collector) -class CustomCollector: - """CustomCollector represents the Prometheus Collector object +class _CustomCollector: + """_CustomCollector represents the Prometheus Collector object https://github.com/prometheus/client_python#custom-collectors """ diff --git a/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py b/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py index b10ac67898a..29ff3e28f63 100644 --- a/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py +++ b/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py @@ -19,7 +19,7 @@ from prometheus_client.core import CounterMetricFamily, GaugeMetricFamily from opentelemetry.exporter.prometheus import ( - CustomCollector, + _CustomCollector, PrometheusMetricExporter, ) from opentelemetry.sdk._metrics.export import MetricExportResult @@ -79,7 +79,7 @@ def test_histogram_to_prometheus(self): attributes={"histo": 1}, ) - collector = CustomCollector("testprefix") + collector = _CustomCollector("testprefix") collector.add_metrics_data([record]) result_bytes = generate_latest(collector) result = result_bytes.decode("utf-8") @@ -95,7 +95,7 @@ def test_sum_to_prometheus(self): description="testdesc", unit="testunit", ) - collector = CustomCollector("testprefix") + collector = _CustomCollector("testprefix") collector.add_metrics_data([record]) for prometheus_metric in collector.collect(): @@ -123,7 +123,7 @@ def test_gauge_to_prometheus(self): description="testdesc", unit="testunit", ) - collector = CustomCollector("testprefix") + collector = _CustomCollector("testprefix") collector.add_metrics_data([record]) for prometheus_metric in collector.collect(): @@ -148,13 +148,13 @@ def test_invalid_metric(self): description="testdesc", unit="testunit", ) - collector = CustomCollector("testprefix") + collector = _CustomCollector("testprefix") collector.add_metrics_data([record]) collector.collect() self.assertLogs("opentelemetry.exporter.prometheus", level="WARNING") def test_sanitize(self): - collector = CustomCollector("testprefix") + collector = _CustomCollector("testprefix") self.assertEqual( collector._sanitize("1!2@3#4$5%6^7&8*9(0)_-"), "1_2_3_4_5_6_7_8_9_0___", From 57a47e601d50920000649416d0bee0bebbade1b9 Mon Sep 17 00:00:00 2001 From: Alex Boten Date: Mon, 28 Feb 2022 09:20:49 -0800 Subject: [PATCH 13/31] move prom exporter to prerelease --- eachdist.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eachdist.ini b/eachdist.ini index 576d46a5fc7..f65be079bec 100644 --- a/eachdist.ini +++ b/eachdist.ini @@ -35,6 +35,7 @@ version=0.28b1 packages= opentelemetry-opentracing-shim opentelemetry-exporter-opencensus + opentelemetry-exporter-prometheus opentelemetry-distro opentelemetry-semantic-conventions opentelemetry-test-utils @@ -45,7 +46,6 @@ version=1.10a0 packages= opentelemetry-exporter-prometheus-remote-write - opentelemetry-exporter-prometheus opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp-proto-grpc From 629b9370d6265c81df0369456e8975acf2770dfc Mon Sep 17 00:00:00 2001 From: Alex Boten Date: Mon, 28 Feb 2022 09:20:58 -0800 Subject: [PATCH 14/31] bump api/sdk dep --- exporter/opentelemetry-exporter-prometheus/setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exporter/opentelemetry-exporter-prometheus/setup.cfg b/exporter/opentelemetry-exporter-prometheus/setup.cfg index 68f953d92d0..65a018d374e 100644 --- a/exporter/opentelemetry-exporter-prometheus/setup.cfg +++ b/exporter/opentelemetry-exporter-prometheus/setup.cfg @@ -41,8 +41,8 @@ package_dir= packages=find_namespace: install_requires = prometheus_client >= 0.5.0, < 1.0.0 - opentelemetry-api >= 1.7.1 - opentelemetry-sdk >= 1.7.1 + opentelemetry-api >= 1.9.1 + opentelemetry-sdk >= 1.9.1 [options.packages.find] where = src From 680e662caed0b4fe220bde6792279f8db8fa5ae8 Mon Sep 17 00:00:00 2001 From: Alex Boten Date: Mon, 28 Feb 2022 12:10:37 -0800 Subject: [PATCH 15/31] implement pull metric reader --- .../exporter/prometheus/__init__.py | 37 +++++++++++-------- .../tests/test_prometheus_exporter.py | 12 +++--- .../sdk/_metrics/export/__init__.py | 33 +++++++++++++++++ 3 files changed, 60 insertions(+), 22 deletions(-) diff --git a/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py b/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py index 387ac32de44..e9a5fad20f3 100644 --- a/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py +++ b/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py @@ -27,28 +27,29 @@ .. code:: python - from opentelemetry import metrics - from opentelemetry.exporter.prometheus import PrometheusMetricExporter - from opentelemetry.sdk.metrics import Meter from prometheus_client import start_http_server + from opentelemetry._metrics import get_meter_provider, set_meter_provider + from opentelemetry.exporter.prometheus import PrometheusMetricExporter + from opentelemetry.sdk._metrics import MeterProvider + from opentelemetry.sdk._metrics.export import PullMetricExporterReader + # Start Prometheus client start_http_server(port=8000, addr="localhost") - # Meter is responsible for creating and recording metrics - metrics.set_meter_provider(MeterProvider()) - meter = metrics.get_meter(__name__) # exporter to export metrics to Prometheus prefix = "MyAppPrefix" exporter = PrometheusMetricExporter(prefix) - # Starts the collect/export pipeline for metrics - metrics.get_meter_provider().start_pipeline(meter, exporter, 5) + reader = PullMetricExporterReader(exporter) + + # Meter is responsible for creating and recording metrics + set_meter_provider(MeterProvider(metric_readers=[reader])) + meter = get_meter_provider().get_meter(__name__) counter = meter.create_counter( "requests", - "number of requests", "requests", - int, + "number of requests", ) # Labels are used to identify key-values that are associated with a specific @@ -71,12 +72,12 @@ from prometheus_client import core from opentelemetry.sdk._metrics.export import ( - MetricExporter, MetricExportResult, + PullMetricExporter, ) from opentelemetry.sdk._metrics.point import Gauge, Histogram, Metric, Sum -logger = logging.getLogger(__name__) +_logger = logging.getLogger(__name__) def _convert_buckets(metric: Metric) -> Sequence[Tuple[str, int]]: @@ -93,7 +94,7 @@ def _convert_buckets(metric: Metric) -> Sequence[Tuple[str, int]]: return buckets -class PrometheusMetricExporter(MetricExporter): +class PrometheusMetricExporter(PullMetricExporter): """Prometheus metric exporter for OpenTelemetry. Args: @@ -102,7 +103,8 @@ class PrometheusMetricExporter(MetricExporter): """ def __init__(self, prefix: str = ""): - self._collector = _CustomCollector(prefix) + self._collect = None + self._collector = _CustomCollector(self.collect, prefix) core.REGISTRY.register(self._collector) def export(self, metrics: Sequence[Metric]) -> MetricExportResult: @@ -118,8 +120,9 @@ class _CustomCollector: https://github.com/prometheus/client_python#custom-collectors """ - def __init__(self, prefix: str = ""): + def __init__(self, callback, prefix: str = ""): self._prefix = prefix + self._callback = callback self._metrics_to_export = collections.deque() self._non_letters_nor_digits_re = re.compile( r"[^\w]", re.UNICODE | re.IGNORECASE @@ -135,6 +138,8 @@ def collect(self) -> None: Collect is invoked every time a prometheus.Gatherer is run for example when the HTTP endpoint is invoked by Prometheus. """ + if self._callback: + self._callback() while self._metrics_to_export: for export_record in self._metrics_to_export.popleft(): @@ -193,7 +198,7 @@ def _translate_to_prometheus( labels=label_values, buckets=buckets, sum_value=value ) else: - logger.warning("Unsupported metric type. %s", type(metric.point)) + _logger.warning("Unsupported metric type. %s", type(metric.point)) return prometheus_metric def _sanitize(self, key: str) -> str: diff --git a/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py b/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py index 29ff3e28f63..31a97e9729a 100644 --- a/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py +++ b/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py @@ -19,8 +19,8 @@ from prometheus_client.core import CounterMetricFamily, GaugeMetricFamily from opentelemetry.exporter.prometheus import ( - _CustomCollector, PrometheusMetricExporter, + _CustomCollector, ) from opentelemetry.sdk._metrics.export import MetricExportResult from opentelemetry.sdk._metrics.point import AggregationTemporality, Histogram @@ -79,7 +79,7 @@ def test_histogram_to_prometheus(self): attributes={"histo": 1}, ) - collector = _CustomCollector("testprefix") + collector = _CustomCollector(mock.Mock(), "testprefix") collector.add_metrics_data([record]) result_bytes = generate_latest(collector) result = result_bytes.decode("utf-8") @@ -95,7 +95,7 @@ def test_sum_to_prometheus(self): description="testdesc", unit="testunit", ) - collector = _CustomCollector("testprefix") + collector = _CustomCollector(mock.Mock(), "testprefix") collector.add_metrics_data([record]) for prometheus_metric in collector.collect(): @@ -123,7 +123,7 @@ def test_gauge_to_prometheus(self): description="testdesc", unit="testunit", ) - collector = _CustomCollector("testprefix") + collector = _CustomCollector(mock.Mock(), "testprefix") collector.add_metrics_data([record]) for prometheus_metric in collector.collect(): @@ -148,13 +148,13 @@ def test_invalid_metric(self): description="testdesc", unit="testunit", ) - collector = _CustomCollector("testprefix") + collector = _CustomCollector(mock.Mock(), "testprefix") collector.add_metrics_data([record]) collector.collect() self.assertLogs("opentelemetry.exporter.prometheus", level="WARNING") def test_sanitize(self): - collector = _CustomCollector("testprefix") + collector = _CustomCollector(mock.Mock(), "testprefix") self.assertEqual( collector._sanitize("1!2@3#4$5%6^7&8*9(0)_-"), "1_2_3_4_5_6_7_8_9_0___", diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/export/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/export/__init__.py index 284629330f8..d75645b7da3 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/export/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/export/__init__.py @@ -21,6 +21,8 @@ from threading import Event, Thread from typing import IO, Callable, Iterable, Optional, Sequence +from typing_extensions import final + from opentelemetry.context import ( _SUPPRESS_INSTRUMENTATION_KEY, attach, @@ -176,3 +178,34 @@ def _shutdown(): self._daemon_thread.join() self._exporter.shutdown() return True + + +class PullMetricExporter(MetricExporter): + @final + def collect(self) -> None: + if self._collect: + self._collect() + + +class PullMetricExporterReader(MetricReader): + def __init__( + self, + exporter: PullMetricExporter, + ): + super().__init__(preferred_temporality=exporter.preferred_temporality) + self._exporter = exporter + self._exporter._collect = self.collect + + def _receive_metrics(self, metrics: Iterable[Metric]) -> None: + if metrics is None: + return + token = attach(set_value(_SUPPRESS_INSTRUMENTATION_KEY, True)) + try: + self._exporter.export(metrics) + except Exception as e: # pylint: disable=broad-except,invalid-name + _logger.exception("Exception while exporting metrics %s", str(e)) + detach(token) + + def shutdown(self) -> bool: + self._exporter.shutdown() + return True From 0273486316e1b78e7194845e8ba85e03e42cc6e3 Mon Sep 17 00:00:00 2001 From: Alex Boten Date: Mon, 28 Feb 2022 12:11:33 -0800 Subject: [PATCH 16/31] update dep --- exporter/opentelemetry-exporter-prometheus/setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exporter/opentelemetry-exporter-prometheus/setup.cfg b/exporter/opentelemetry-exporter-prometheus/setup.cfg index 65a018d374e..62c21117dcd 100644 --- a/exporter/opentelemetry-exporter-prometheus/setup.cfg +++ b/exporter/opentelemetry-exporter-prometheus/setup.cfg @@ -41,8 +41,8 @@ package_dir= packages=find_namespace: install_requires = prometheus_client >= 0.5.0, < 1.0.0 - opentelemetry-api >= 1.9.1 - opentelemetry-sdk >= 1.9.1 + opentelemetry-api > 1.9.1 + opentelemetry-sdk > 1.9.1 [options.packages.find] where = src From b3d9dea70a4456f9abbee96baea57b8dad97edb9 Mon Sep 17 00:00:00 2001 From: Alex Boten Date: Mon, 28 Feb 2022 13:55:09 -0800 Subject: [PATCH 17/31] update dep --- exporter/opentelemetry-exporter-prometheus/setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exporter/opentelemetry-exporter-prometheus/setup.cfg b/exporter/opentelemetry-exporter-prometheus/setup.cfg index 62c21117dcd..65a018d374e 100644 --- a/exporter/opentelemetry-exporter-prometheus/setup.cfg +++ b/exporter/opentelemetry-exporter-prometheus/setup.cfg @@ -41,8 +41,8 @@ package_dir= packages=find_namespace: install_requires = prometheus_client >= 0.5.0, < 1.0.0 - opentelemetry-api > 1.9.1 - opentelemetry-sdk > 1.9.1 + opentelemetry-api >= 1.9.1 + opentelemetry-sdk >= 1.9.1 [options.packages.find] where = src From b72fcb8acf9d2a3828fd383fa659031b43de3f24 Mon Sep 17 00:00:00 2001 From: Alex Boten Date: Mon, 28 Feb 2022 14:55:55 -0800 Subject: [PATCH 18/31] fix open call --- exporter/opentelemetry-exporter-prometheus/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporter/opentelemetry-exporter-prometheus/setup.py b/exporter/opentelemetry-exporter-prometheus/setup.py index 86067a2bd56..091ba03ca9f 100644 --- a/exporter/opentelemetry-exporter-prometheus/setup.py +++ b/exporter/opentelemetry-exporter-prometheus/setup.py @@ -20,7 +20,7 @@ BASE_DIR, "src", "opentelemetry", "exporter", "prometheus", "version.py" ) PACKAGE_INFO = {} -with open(VERSION_FILENAME) as f: +with open(VERSION_FILENAME, encoding="utf-8") as f: exec(f.read(), PACKAGE_INFO) setuptools.setup(version=PACKAGE_INFO["__version__"]) From 91e857273f0b37122bf7cf91e3d0d2356ac1e5e6 Mon Sep 17 00:00:00 2001 From: Alex Boten Date: Mon, 28 Feb 2022 14:56:30 -0800 Subject: [PATCH 19/31] follow same pattern as metric reader --- .../exporter/prometheus/__init__.py | 4 ++-- .../sdk/_metrics/export/__init__.py | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py b/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py index e9a5fad20f3..fdabac67d54 100644 --- a/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py +++ b/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py @@ -102,8 +102,8 @@ class PrometheusMetricExporter(PullMetricExporter): the metric belongs to. """ - def __init__(self, prefix: str = ""): - self._collect = None + def __init__(self, prefix: str = "") -> None: + super().__init__() self._collector = _CustomCollector(self.collect, prefix) core.REGISTRY.register(self._collector) diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/export/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/export/__init__.py index d75645b7da3..5396e1421dd 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/export/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/export/__init__.py @@ -181,6 +181,21 @@ def _shutdown(): class PullMetricExporter(MetricExporter): + def __init__(self) -> None: + self._collect: Callable[ + ["PullMetricExporter"], + ] = None + + @final + def _set_collect_callback( + self, + func: Callable[ + ["PullMetricExporter"], + ], + ) -> None: + """This function is internal to the SDK. It should not be called or overriden by users""" + self._collect = func + @final def collect(self) -> None: if self._collect: @@ -194,7 +209,7 @@ def __init__( ): super().__init__(preferred_temporality=exporter.preferred_temporality) self._exporter = exporter - self._exporter._collect = self.collect + self._exporter._set_collect_callback(self.collect) def _receive_metrics(self, metrics: Iterable[Metric]) -> None: if metrics is None: From 910052336d6b61d08bbe6b95dbbd36ddae628bbb Mon Sep 17 00:00:00 2001 From: Alex Boten Date: Mon, 28 Feb 2022 15:00:11 -0800 Subject: [PATCH 20/31] fix callable --- .../src/opentelemetry/sdk/_metrics/export/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/export/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/export/__init__.py index 5396e1421dd..5953f9f6ef4 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/export/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/export/__init__.py @@ -183,14 +183,14 @@ def _shutdown(): class PullMetricExporter(MetricExporter): def __init__(self) -> None: self._collect: Callable[ - ["PullMetricExporter"], + ["PullMetricExporter"], None, ] = None @final def _set_collect_callback( self, func: Callable[ - ["PullMetricExporter"], + ["PullMetricExporter"], None, ], ) -> None: """This function is internal to the SDK. It should not be called or overriden by users""" From 89622a9ed7c27a6e14b636405e4d1b55158d5c49 Mon Sep 17 00:00:00 2001 From: Alex Boten Date: Mon, 28 Feb 2022 15:06:21 -0800 Subject: [PATCH 21/31] remove unnecessary public vars --- .../opentelemetry-exporter-prometheus/setup.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/exporter/opentelemetry-exporter-prometheus/setup.py b/exporter/opentelemetry-exporter-prometheus/setup.py index 091ba03ca9f..9fdbd7de497 100644 --- a/exporter/opentelemetry-exporter-prometheus/setup.py +++ b/exporter/opentelemetry-exporter-prometheus/setup.py @@ -15,12 +15,12 @@ import setuptools -BASE_DIR = os.path.dirname(__file__) -VERSION_FILENAME = os.path.join( - BASE_DIR, "src", "opentelemetry", "exporter", "prometheus", "version.py" +_BASE_DIR = os.path.dirname(__file__) +_VERSION_FILENAME = os.path.join( + _BASE_DIR, "src", "opentelemetry", "exporter", "prometheus", "version.py" ) -PACKAGE_INFO = {} -with open(VERSION_FILENAME, encoding="utf-8") as f: - exec(f.read(), PACKAGE_INFO) +_PACKAGE_INFO = {} +with open(_VERSION_FILENAME, encoding="utf-8") as f: + exec(f.read(), _PACKAGE_INFO) -setuptools.setup(version=PACKAGE_INFO["__version__"]) +setuptools.setup(version=_PACKAGE_INFO["__version__"]) From d381376e8b5827c94b87d058a123045ad00d5bd8 Mon Sep 17 00:00:00 2001 From: Alex Boten Date: Tue, 1 Mar 2022 12:44:37 -0800 Subject: [PATCH 22/31] Update exporter/opentelemetry-exporter-prometheus/LICENSE Co-authored-by: Diego Hurtado --- exporter/opentelemetry-exporter-prometheus/LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporter/opentelemetry-exporter-prometheus/LICENSE b/exporter/opentelemetry-exporter-prometheus/LICENSE index 261eeb9e9f8..1ef7dad2c5c 100644 --- a/exporter/opentelemetry-exporter-prometheus/LICENSE +++ b/exporter/opentelemetry-exporter-prometheus/LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright The OpenTelemetry Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. From 29fee5d4ee49e92f1e57b0b411d766daf6dfb034 Mon Sep 17 00:00:00 2001 From: Alex Boten Date: Wed, 2 Mar 2022 15:50:57 -0800 Subject: [PATCH 23/31] Apply suggestions from code review Co-authored-by: Diego Hurtado Co-authored-by: Nathaniel Ruiz Nowell --- .../src/opentelemetry/exporter/prometheus/__init__.py | 8 +++++--- .../src/opentelemetry/sdk/_metrics/export/__init__.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py b/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py index fdabac67d54..4544f11a0b7 100644 --- a/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py +++ b/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py @@ -37,7 +37,7 @@ # Start Prometheus client start_http_server(port=8000, addr="localhost") - # exporter to export metrics to Prometheus + # Exporter to export metrics to Prometheus prefix = "MyAppPrefix" exporter = PrometheusMetricExporter(prefix) reader = PullMetricExporterReader(exporter) @@ -117,6 +117,8 @@ def shutdown(self) -> None: class _CustomCollector: """_CustomCollector represents the Prometheus Collector object + + See more: https://github.com/prometheus/client_python#custom-collectors """ @@ -135,10 +137,10 @@ def add_metrics_data(self, export_records: Sequence[Metric]) -> None: def collect(self) -> None: """Collect fetches the metrics from OpenTelemetry and delivers them as Prometheus Metrics. - Collect is invoked every time a prometheus.Gatherer is run + Collect is invoked every time a ``prometheus.Gatherer`` is run for example when the HTTP endpoint is invoked by Prometheus. """ - if self._callback: + if self._callback is not None: self._callback() while self._metrics_to_export: diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/export/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/export/__init__.py index 5953f9f6ef4..ff71fd7cf11 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/export/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/export/__init__.py @@ -198,7 +198,7 @@ def _set_collect_callback( @final def collect(self) -> None: - if self._collect: + if self._collect is not None: self._collect() From a4cc1912e0bda34f3eaa0d4d6d9289befc67b851 Mon Sep 17 00:00:00 2001 From: Alex Boten Date: Thu, 3 Mar 2022 09:53:36 -0800 Subject: [PATCH 24/31] feedback from reviews --- .../src/opentelemetry/exporter/prometheus/__init__.py | 10 +++++----- .../src/opentelemetry/sdk/_metrics/export/__init__.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py b/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py index 4544f11a0b7..eff7b4086e8 100644 --- a/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py +++ b/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py @@ -32,7 +32,7 @@ from opentelemetry._metrics import get_meter_provider, set_meter_provider from opentelemetry.exporter.prometheus import PrometheusMetricExporter from opentelemetry.sdk._metrics import MeterProvider - from opentelemetry.sdk._metrics.export import PullMetricExporterReader + from opentelemetry.sdk._metrics.export import PullingMetricReader # Start Prometheus client start_http_server(port=8000, addr="localhost") @@ -40,11 +40,11 @@ # Exporter to export metrics to Prometheus prefix = "MyAppPrefix" exporter = PrometheusMetricExporter(prefix) - reader = PullMetricExporterReader(exporter) + reader = PullingMetricReader(exporter) # Meter is responsible for creating and recording metrics set_meter_provider(MeterProvider(metric_readers=[reader])) - meter = get_meter_provider().get_meter(__name__) + meter = get_meter_provider().get_meter("myapp", "0.1.2") counter = meter.create_counter( "requests", @@ -126,7 +126,7 @@ def __init__(self, callback, prefix: str = ""): self._prefix = prefix self._callback = callback self._metrics_to_export = collections.deque() - self._non_letters_nor_digits_re = re.compile( + self._non_letters_digits_underscore_re = re.compile( r"[^\w]", re.UNICODE | re.IGNORECASE ) @@ -207,4 +207,4 @@ def _sanitize(self, key: str) -> str: """sanitize the given metric name or label according to Prometheus rule. Replace all characters other than [A-Za-z0-9_] with '_'. """ - return self._non_letters_nor_digits_re.sub("_", key) + return self._non_letters_digits_underscore_re.sub("_", key) diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/export/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/export/__init__.py index ff71fd7cf11..b3576044dfe 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/export/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/export/__init__.py @@ -202,7 +202,7 @@ def collect(self) -> None: self._collect() -class PullMetricExporterReader(MetricReader): +class PullingMetricReader(MetricReader): def __init__( self, exporter: PullMetricExporter, From 6673bec631054fcc52fdcbe0e7b0e222d24841e8 Mon Sep 17 00:00:00 2001 From: Alex Boten Date: Thu, 3 Mar 2022 12:36:19 -0800 Subject: [PATCH 25/31] as per feedback in today SIG simplified the implementation to remove unnecessary interfaces. --- .../exporter/prometheus/__init__.py | 35 ++++++++------ .../sdk/_metrics/export/__init__.py | 46 ------------------- 2 files changed, 22 insertions(+), 59 deletions(-) diff --git a/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py b/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py index eff7b4086e8..6480fcd2c6c 100644 --- a/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py +++ b/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py @@ -39,8 +39,7 @@ # Exporter to export metrics to Prometheus prefix = "MyAppPrefix" - exporter = PrometheusMetricExporter(prefix) - reader = PullingMetricReader(exporter) + reader = PrometheusMetricReader(prefix) # Meter is responsible for creating and recording metrics set_meter_provider(MeterProvider(metric_readers=[reader])) @@ -67,14 +66,17 @@ import collections import logging import re -from typing import Optional, Sequence, Tuple +from typing import Iterable, Optional, Sequence, Tuple from prometheus_client import core -from opentelemetry.sdk._metrics.export import ( - MetricExportResult, - PullMetricExporter, +from opentelemetry.context import ( + _SUPPRESS_INSTRUMENTATION_KEY, + attach, + detach, + set_value, ) +from opentelemetry.sdk._metrics.export import MetricReader from opentelemetry.sdk._metrics.point import Gauge, Histogram, Metric, Sum _logger = logging.getLogger(__name__) @@ -94,7 +96,7 @@ def _convert_buckets(metric: Metric) -> Sequence[Tuple[str, int]]: return buckets -class PrometheusMetricExporter(PullMetricExporter): +class PrometheusMetricReader(MetricReader): """Prometheus metric exporter for OpenTelemetry. Args: @@ -107,17 +109,24 @@ def __init__(self, prefix: str = "") -> None: self._collector = _CustomCollector(self.collect, prefix) core.REGISTRY.register(self._collector) - def export(self, metrics: Sequence[Metric]) -> MetricExportResult: - self._collector.add_metrics_data(metrics) - return MetricExportResult.SUCCESS - - def shutdown(self) -> None: + def _receive_metrics(self, metrics: Iterable[Metric]) -> None: + if metrics is None: + return + token = attach(set_value(_SUPPRESS_INSTRUMENTATION_KEY, True)) + try: + self._collector.add_metrics_data(metrics) + except Exception as e: # pylint: disable=broad-except,invalid-name + _logger.exception("Exception while exporting metrics %s", str(e)) + detach(token) + + def shutdown(self) -> bool: core.REGISTRY.unregister(self._collector) + return True class _CustomCollector: """_CustomCollector represents the Prometheus Collector object - + See more: https://github.com/prometheus/client_python#custom-collectors """ diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/export/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/export/__init__.py index b3576044dfe..1144bf97f51 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/export/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/export/__init__.py @@ -178,49 +178,3 @@ def _shutdown(): self._daemon_thread.join() self._exporter.shutdown() return True - - -class PullMetricExporter(MetricExporter): - def __init__(self) -> None: - self._collect: Callable[ - ["PullMetricExporter"], None, - ] = None - - @final - def _set_collect_callback( - self, - func: Callable[ - ["PullMetricExporter"], None, - ], - ) -> None: - """This function is internal to the SDK. It should not be called or overriden by users""" - self._collect = func - - @final - def collect(self) -> None: - if self._collect is not None: - self._collect() - - -class PullingMetricReader(MetricReader): - def __init__( - self, - exporter: PullMetricExporter, - ): - super().__init__(preferred_temporality=exporter.preferred_temporality) - self._exporter = exporter - self._exporter._set_collect_callback(self.collect) - - def _receive_metrics(self, metrics: Iterable[Metric]) -> None: - if metrics is None: - return - token = attach(set_value(_SUPPRESS_INSTRUMENTATION_KEY, True)) - try: - self._exporter.export(metrics) - except Exception as e: # pylint: disable=broad-except,invalid-name - _logger.exception("Exception while exporting metrics %s", str(e)) - detach(token) - - def shutdown(self) -> bool: - self._exporter.shutdown() - return True From e695b0ef587fb9cda5fe2408ba709039df92fe45 Mon Sep 17 00:00:00 2001 From: Alex Boten Date: Thu, 3 Mar 2022 13:30:31 -0800 Subject: [PATCH 26/31] fix tests --- .../opentelemetry-exporter-prometheus/setup.cfg | 4 ++-- .../exporter/prometheus/__init__.py | 3 +-- .../tests/test_prometheus_exporter.py | 17 ++++------------- 3 files changed, 7 insertions(+), 17 deletions(-) diff --git a/exporter/opentelemetry-exporter-prometheus/setup.cfg b/exporter/opentelemetry-exporter-prometheus/setup.cfg index 65a018d374e..53e9ef17ede 100644 --- a/exporter/opentelemetry-exporter-prometheus/setup.cfg +++ b/exporter/opentelemetry-exporter-prometheus/setup.cfg @@ -51,5 +51,5 @@ where = src test = [options.entry_points] -opentelemetry_exporter = - prometheus = opentelemetry.exporter.prometheus:PrometheusMetricExporter +opentelemetry_metric_reader = + prometheus = opentelemetry.exporter.prometheus:PrometheusMetricReader diff --git a/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py b/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py index 6480fcd2c6c..83f28973ee7 100644 --- a/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py +++ b/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py @@ -30,9 +30,8 @@ from prometheus_client import start_http_server from opentelemetry._metrics import get_meter_provider, set_meter_provider - from opentelemetry.exporter.prometheus import PrometheusMetricExporter + from opentelemetry.exporter.prometheus import PrometheusMetricReader from opentelemetry.sdk._metrics import MeterProvider - from opentelemetry.sdk._metrics.export import PullingMetricReader # Start Prometheus client start_http_server(port=8000, addr="localhost") diff --git a/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py b/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py index 31a97e9729a..d2d22f5c79f 100644 --- a/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py +++ b/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py @@ -19,7 +19,7 @@ from prometheus_client.core import CounterMetricFamily, GaugeMetricFamily from opentelemetry.exporter.prometheus import ( - PrometheusMetricExporter, + PrometheusMetricReader, _CustomCollector, ) from opentelemetry.sdk._metrics.export import MetricExportResult @@ -32,7 +32,7 @@ ) -class TestPrometheusMetricExporter(unittest.TestCase): +class TestPrometheusMetricReader(unittest.TestCase): def setUp(self): self._mock_registry_register = mock.Mock() self._registry_register_patch = mock.patch( @@ -44,7 +44,7 @@ def setUp(self): def test_constructor(self): """Test the constructor.""" with self._registry_register_patch: - exporter = PrometheusMetricExporter("testprefix") + exporter = PrometheusMetricReader("testprefix") self.assertEqual(exporter._collector._prefix, "testprefix") self.assertTrue(self._mock_registry_register.called) @@ -52,19 +52,10 @@ def test_shutdown(self): with mock.patch( "prometheus_client.core.REGISTRY.unregister" ) as registry_unregister_patch: - exporter = PrometheusMetricExporter() + exporter = PrometheusMetricReader() exporter.shutdown() self.assertTrue(registry_unregister_patch.called) - def test_export(self): - with self._registry_register_patch: - record = _generate_sum("sum_int", 33) - exporter = PrometheusMetricExporter() - result = exporter.export([record]) - # pylint: disable=protected-access - self.assertEqual(len(exporter._collector._metrics_to_export), 1) - self.assertIs(result, MetricExportResult.SUCCESS) - def test_histogram_to_prometheus(self): record = _generate_metric( "test@name", From 0cc5644f35a6e03b287ecc8033e83b56ba42547c Mon Sep 17 00:00:00 2001 From: Alex Boten Date: Thu, 3 Mar 2022 13:43:23 -0800 Subject: [PATCH 27/31] fix lint --- .../src/opentelemetry/sdk/_metrics/export/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/export/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/export/__init__.py index 1144bf97f51..284629330f8 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/export/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/export/__init__.py @@ -21,8 +21,6 @@ from threading import Event, Thread from typing import IO, Callable, Iterable, Optional, Sequence -from typing_extensions import final - from opentelemetry.context import ( _SUPPRESS_INSTRUMENTATION_KEY, attach, From f4097688cd9546ea2c7f84ea8f4076063d2b127c Mon Sep 17 00:00:00 2001 From: Alex Boten Date: Thu, 3 Mar 2022 14:24:14 -0800 Subject: [PATCH 28/31] fix lint --- .../tests/test_prometheus_exporter.py | 1 - 1 file changed, 1 deletion(-) diff --git a/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py b/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py index d2d22f5c79f..f676289220f 100644 --- a/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py +++ b/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py @@ -22,7 +22,6 @@ PrometheusMetricReader, _CustomCollector, ) -from opentelemetry.sdk._metrics.export import MetricExportResult from opentelemetry.sdk._metrics.point import AggregationTemporality, Histogram from opentelemetry.test.metrictestutil import ( _generate_gauge, From d52918f98ac22cb0f5a4d451b5553c4266aed88b Mon Sep 17 00:00:00 2001 From: Alex Boten Date: Thu, 3 Mar 2022 15:01:08 -0800 Subject: [PATCH 29/31] more cleanup --- .../opentelemetry/exporter/prometheus/__init__.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py b/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py index 83f28973ee7..c7cedc5a058 100644 --- a/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py +++ b/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py @@ -69,12 +69,6 @@ from prometheus_client import core -from opentelemetry.context import ( - _SUPPRESS_INSTRUMENTATION_KEY, - attach, - detach, - set_value, -) from opentelemetry.sdk._metrics.export import MetricReader from opentelemetry.sdk._metrics.point import Gauge, Histogram, Metric, Sum @@ -111,12 +105,7 @@ def __init__(self, prefix: str = "") -> None: def _receive_metrics(self, metrics: Iterable[Metric]) -> None: if metrics is None: return - token = attach(set_value(_SUPPRESS_INSTRUMENTATION_KEY, True)) - try: - self._collector.add_metrics_data(metrics) - except Exception as e: # pylint: disable=broad-except,invalid-name - _logger.exception("Exception while exporting metrics %s", str(e)) - detach(token) + self._collector.add_metrics_data(metrics) def shutdown(self) -> bool: core.REGISTRY.unregister(self._collector) From 088c53bc7c3faba96b55b9c9a7a10892f2a2f1b7 Mon Sep 17 00:00:00 2001 From: Alex Boten Date: Thu, 3 Mar 2022 15:17:49 -0800 Subject: [PATCH 30/31] more cleanup --- .../src/opentelemetry/exporter/prometheus/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py b/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py index c7cedc5a058..a952e56bcf8 100644 --- a/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py +++ b/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py @@ -99,8 +99,9 @@ class PrometheusMetricReader(MetricReader): def __init__(self, prefix: str = "") -> None: super().__init__() - self._collector = _CustomCollector(self.collect, prefix) + self._collector = _CustomCollector(prefix) core.REGISTRY.register(self._collector) + self._collector._callback = self.collect def _receive_metrics(self, metrics: Iterable[Metric]) -> None: if metrics is None: @@ -119,9 +120,9 @@ class _CustomCollector: https://github.com/prometheus/client_python#custom-collectors """ - def __init__(self, callback, prefix: str = ""): + def __init__(self, prefix: str = ""): self._prefix = prefix - self._callback = callback + self._callback = None self._metrics_to_export = collections.deque() self._non_letters_digits_underscore_re = re.compile( r"[^\w]", re.UNICODE | re.IGNORECASE From 7c7709df22d72291a3eb2cdc0f12d42aa319102e Mon Sep 17 00:00:00 2001 From: Alex Boten Date: Thu, 3 Mar 2022 15:40:03 -0800 Subject: [PATCH 31/31] fix tests --- .../tests/test_prometheus_exporter.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py b/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py index f676289220f..af4bc7c0fdd 100644 --- a/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py +++ b/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py @@ -69,7 +69,7 @@ def test_histogram_to_prometheus(self): attributes={"histo": 1}, ) - collector = _CustomCollector(mock.Mock(), "testprefix") + collector = _CustomCollector("testprefix") collector.add_metrics_data([record]) result_bytes = generate_latest(collector) result = result_bytes.decode("utf-8") @@ -85,7 +85,7 @@ def test_sum_to_prometheus(self): description="testdesc", unit="testunit", ) - collector = _CustomCollector(mock.Mock(), "testprefix") + collector = _CustomCollector("testprefix") collector.add_metrics_data([record]) for prometheus_metric in collector.collect(): @@ -113,7 +113,7 @@ def test_gauge_to_prometheus(self): description="testdesc", unit="testunit", ) - collector = _CustomCollector(mock.Mock(), "testprefix") + collector = _CustomCollector("testprefix") collector.add_metrics_data([record]) for prometheus_metric in collector.collect(): @@ -138,13 +138,13 @@ def test_invalid_metric(self): description="testdesc", unit="testunit", ) - collector = _CustomCollector(mock.Mock(), "testprefix") + collector = _CustomCollector("testprefix") collector.add_metrics_data([record]) collector.collect() self.assertLogs("opentelemetry.exporter.prometheus", level="WARNING") def test_sanitize(self): - collector = _CustomCollector(mock.Mock(), "testprefix") + collector = _CustomCollector("testprefix") self.assertEqual( collector._sanitize("1!2@3#4$5%6^7&8*9(0)_-"), "1_2_3_4_5_6_7_8_9_0___",