diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 1036cb75400..4fccfe73524 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -25,7 +25,7 @@ jobs: python-version: ${{ env[matrix.python-version] }} architecture: 'x64' - name: Install tox - run: pip install tox==3.27.1 -U tox-factor + run: pip install tox - name: Cache tox environment # Preserves .tox directory between runs for faster installs uses: actions/cache@v2 diff --git a/.github/workflows/public-api-check.yml b/.github/workflows/public-api-check.yml index 46432af0b54..67dcb798310 100644 --- a/.github/workflows/public-api-check.yml +++ b/.github/workflows/public-api-check.yml @@ -34,7 +34,7 @@ jobs: python-version: '3.10' - name: Install tox - run: pip install tox==3.27.1 -U tox-factor + run: pip install tox - name: Public API Check run: tox -e public-symbols-check diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ca1d0fa58fd..0457d584f2f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ env: # Otherwise, set variable to the commit of your branch on # opentelemetry-python-contrib which is compatible with these Core repo # changes. - CONTRIB_REPO_SHA: 2977f143df1d474735e8bdfecd91d92d534e80dc + CONTRIB_REPO_SHA: 1a984d3ba18d4080c58485b7d807dba241179d41 # This is needed because we do not clone the core repo in contrib builds anymore. # When running contrib builds as part of core builds, we use actions/checkout@v2 which # does not set an environment variable (simply just runs tox), which is different when @@ -42,9 +42,6 @@ jobs: - "getting-started" - "opentracing-shim" - "opencensus-shim" - - "exporter-jaeger-combined" - - "exporter-jaeger-proto-grpc" - - "exporter-jaeger-thrift" - "exporter-opencensus" - "exporter-otlp-proto-common" - "exporter-otlp-combined" @@ -58,6 +55,16 @@ jobs: - "propagator-b3" - "propagator-jaeger" os: [ubuntu-20.04, windows-2019] + exclude: + - python-version: pypy3 + package: "opencensus-shim" + - python-version: pypy3 + package: "exporter-opencensus" + - python-version: pypy3 + package: "exporter-otlp-combined" + - python-version: pypy3 + package: "exporter-otlp-proto-grpc" + steps: - name: Checkout Core Repo @ SHA - ${{ github.sha }} uses: actions/checkout@v2 @@ -67,7 +74,7 @@ jobs: python-version: ${{ env[matrix.python-version] }} architecture: 'x64' - name: Install tox - run: pip install tox==3.27.1 -U tox-factor + run: pip install tox - name: Cache tox environment # Preserves .tox directory between runs for faster installs uses: actions/cache@v2 @@ -75,7 +82,7 @@ jobs: path: | .tox ~/.cache/pip - key: v3-tox-cache-${{ env.RUN_MATRIX_COMBINATION }}-${{ hashFiles('tox.ini', + key: v4-tox-cache-${{ env.RUN_MATRIX_COMBINATION }}-${{ hashFiles('tox.ini', 'dev-requirements.txt') }}-core - name: Windows does not let git check out files with long names if: ${{ matrix.os == 'windows-2019'}} @@ -100,7 +107,7 @@ jobs: python-version: '3.10' architecture: 'x64' - name: Install tox - run: pip install tox==3.27.1 + run: pip install tox - name: Cache tox environment # Preserves .tox directory between runs for faster installs uses: actions/cache@v2 @@ -108,7 +115,7 @@ jobs: path: | .tox ~/.cache/pip - key: v3-tox-cache-${{ matrix.tox-environment }}-${{ hashFiles('tox.ini', 'dev-requirements.txt') + key: v4-tox-cache-${{ matrix.tox-environment }}-${{ hashFiles('tox.ini', 'dev-requirements.txt') }}-core - name: run tox run: tox -e ${{ matrix.tox-environment }} @@ -169,7 +176,7 @@ jobs: - "tornado" - "tortoiseorm" - "urllib" - - "urllib3" + - "urllib3v" - "wsgi" - "prometheus-remote-write" - "richconsole" @@ -191,7 +198,7 @@ jobs: python-version: ${{ env[matrix.python-version] }} architecture: 'x64' - name: Install tox - run: pip install tox==3.27.1 -U tox-factor + run: pip install tox - name: Cache tox environment # Preserves .tox directory between runs for faster installs uses: actions/cache@v2 diff --git a/.pylintrc b/.pylintrc index 5b0f7862526..ab11620d772 100644 --- a/.pylintrc +++ b/.pylintrc @@ -76,6 +76,7 @@ disable=missing-docstring, unused-argument, # temp-pylint-upgrade redefined-builtin, cyclic-import, + broad-exception-raised, # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option @@ -480,10 +481,3 @@ max-statements=50 # Minimum number of public methods for a class (see R0903). min-public-methods=2 - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception". -overgeneral-exceptions=Exception diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d814cb8c48..a70ce75eaa2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +- Fix flush error when no LoggerProvider configured for LoggingHandler + ([#3608](https://github.com/open-telemetry/opentelemetry-python/pull/3608)) +- Fix `OTLPMetricExporter` ignores `preferred_aggregation` property + ([#3603](https://github.com/open-telemetry/opentelemetry-python/pull/3603)) +- Logs: set `observed_timestamp` field + ([#3565](https://github.com/open-telemetry/opentelemetry-python/pull/3565)) - Add missing Resource SchemaURL in OTLP exporters ([#3652](https://github.com/open-telemetry/opentelemetry-python/pull/3652)) - Fix loglevel warning text @@ -35,6 +41,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#3645](https://github.com/open-telemetry/opentelemetry-python/pull/3645)) - Add Proxy classes for logging ([#3575](https://github.com/open-telemetry/opentelemetry-python/pull/3575)) +- Remove dependency on 'backoff' library + ([#3679](https://github.com/open-telemetry/opentelemetry-python/pull/3679)) ## Version 1.22.0/0.43b0 (2023-12-15) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 344b5585853..485cb6a0fcc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -38,14 +38,12 @@ during their normal contribution hours. This project uses [tox](https://tox.readthedocs.io) to automate some aspects of development, including testing against multiple Python versions. -To install `tox`, run[^1]: +To install `tox`, run: ```console -$ pip install tox==3.27.1 +$ pip install tox ``` -[^1]: Right now we are experiencing issues with `tox==4.x.y`, so we recommend you use this version. - You can run `tox` with the following arguments: - `tox` to run all existing tox commands, including unit tests for all packages diff --git a/dev-requirements.txt b/dev-requirements.txt index 11adfa75665..f440423ffcc 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -17,5 +17,5 @@ requests==2.31.0 ruamel.yaml==0.17.21 asgiref==3.7.2 psutil==5.9.6 -GitPython==3.1.40 +GitPython==3.1.41 flaky==3.7.0 diff --git a/docs/examples/fork-process-model/flask-gunicorn/requirements.txt b/docs/examples/fork-process-model/flask-gunicorn/requirements.txt index 0323bd5c5eb..ad166e35901 100644 --- a/docs/examples/fork-process-model/flask-gunicorn/requirements.txt +++ b/docs/examples/fork-process-model/flask-gunicorn/requirements.txt @@ -12,7 +12,7 @@ opentelemetry-instrumentation==0.41b0 opentelemetry-instrumentation-flask==0.41b0 opentelemetry-instrumentation-wsgi==0.41b0 opentelemetry-sdk==1.20.0 -protobuf==3.19.4 +protobuf==3.19.5 six==1.15.0 thrift==0.13.0 uWSGI==2.0.22 diff --git a/docs/examples/fork-process-model/flask-uwsgi/requirements.txt b/docs/examples/fork-process-model/flask-uwsgi/requirements.txt index 0323bd5c5eb..ad166e35901 100644 --- a/docs/examples/fork-process-model/flask-uwsgi/requirements.txt +++ b/docs/examples/fork-process-model/flask-uwsgi/requirements.txt @@ -12,7 +12,7 @@ opentelemetry-instrumentation==0.41b0 opentelemetry-instrumentation-flask==0.41b0 opentelemetry-instrumentation-wsgi==0.41b0 opentelemetry-sdk==1.20.0 -protobuf==3.19.4 +protobuf==3.19.5 six==1.15.0 thrift==0.13.0 uWSGI==2.0.22 diff --git a/docs/getting_started/tests/requirements.txt b/docs/getting_started/tests/requirements.txt index 962008c6488..79444476a25 100644 --- a/docs/getting_started/tests/requirements.txt +++ b/docs/getting_started/tests/requirements.txt @@ -10,7 +10,7 @@ idna==3.4 importlib-metadata==6.8.0 iniconfig==2.0.0 itsdangerous==2.1.2 -Jinja2==3.1.2 +Jinja2==3.1.3 MarkupSafe==2.1.3 packaging==23.2 pluggy==1.3.0 @@ -18,7 +18,7 @@ py==1.11.0 py-cpuinfo==9.0.0 pytest==7.1.3 pytest-benchmark==4.0.0 -requests==2.26.0 +requests==2.31.0 tomli==2.0.1 typing_extensions==4.8.0 urllib3==1.26.18 diff --git a/exporter/opentelemetry-exporter-otlp-proto-common/pyproject.toml b/exporter/opentelemetry-exporter-otlp-proto-common/pyproject.toml index e5b3084dbb8..979ffb87c86 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-common/pyproject.toml +++ b/exporter/opentelemetry-exporter-otlp-proto-common/pyproject.toml @@ -25,7 +25,6 @@ classifiers = [ ] dependencies = [ "opentelemetry-proto == 1.23.0.dev", - "backoff >= 1.10.0, < 3.0.0; python_version>='3.8'", ] [project.optional-dependencies] diff --git a/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py b/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py index bd6ca4ad180..6593d89fd87 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py +++ b/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py @@ -15,9 +15,17 @@ import logging from collections.abc import Sequence -from typing import Any, Mapping, Optional, List, Callable, TypeVar, Dict - -import backoff +from itertools import count +from typing import ( + Any, + Mapping, + Optional, + List, + Callable, + TypeVar, + Dict, + Iterator, +) from opentelemetry.sdk.util.instrumentation import InstrumentationScope from opentelemetry.proto.common.v1.common_pb2 import ( @@ -37,7 +45,6 @@ from opentelemetry.sdk.trace import Resource from opentelemetry.util.types import Attributes - _logger = logging.getLogger(__name__) _TypingResourceT = TypeVar("_TypingResourceT") @@ -113,7 +120,6 @@ def _get_resource_data( resource_class: Callable[..., _TypingResourceT], name: str, ) -> List[_TypingResourceT]: - resource_data = [] for ( @@ -134,14 +140,36 @@ def _get_resource_data( return resource_data -# Work around API change between backoff 1.x and 2.x. Since 2.0.0 the backoff -# wait generator API requires a first .send(None) before reading the backoff -# values from the generator. -_is_backoff_v2 = next(backoff.expo()) is None - - -def _create_exp_backoff_generator(*args, **kwargs): - gen = backoff.expo(*args, **kwargs) - if _is_backoff_v2: - gen.send(None) - return gen +def _create_exp_backoff_generator(max_value: int = 0) -> Iterator[int]: + """ + Generates an infinite sequence of exponential backoff values. The sequence starts + from 1 (2^0) and doubles each time (2^1, 2^2, 2^3, ...). If a max_value is specified + and non-zero, the generated values will not exceed this maximum, capping at max_value + instead of growing indefinitely. + + Parameters: + - max_value (int, optional): The maximum value to yield. If 0 or not provided, the + sequence grows without bound. + + Returns: + Iterator[int]: An iterator that yields the exponential backoff values, either uncapped or + capped at max_value. + + Example: + ``` + gen = _create_exp_backoff_generator(max_value=10) + for _ in range(5): + print(next(gen)) + ``` + This will print: + 1 + 2 + 4 + 8 + 10 + + Note: this functionality used to be handled by the 'backoff' package. + """ + for i in count(0): + out = 2**i + yield min(out, max_value) if max_value else out diff --git a/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/metrics_encoder/__init__.py b/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/metrics_encoder/__init__.py index ecd20b8145a..0d66fd28b70 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/metrics_encoder/__init__.py +++ b/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common/_internal/metrics_encoder/__init__.py @@ -16,6 +16,7 @@ from opentelemetry.sdk.metrics.export import ( MetricExporter, ) +from opentelemetry.sdk.metrics.view import Aggregation from os import environ from opentelemetry.sdk.metrics import ( Counter, @@ -65,9 +66,18 @@ class OTLPMetricExporterMixin: def _common_configuration( self, preferred_temporality: Dict[type, AggregationTemporality] = None, + preferred_aggregation: Dict[type, Aggregation] = None, ) -> None: - instrument_class_temporality = {} + MetricExporter.__init__( + self, + preferred_temporality=self._get_temporality(preferred_temporality), + preferred_aggregation=self._get_aggregation(preferred_aggregation), + ) + + def _get_temporality( + self, preferred_temporality: Dict[type, AggregationTemporality] + ) -> Dict[type, AggregationTemporality]: otel_exporter_otlp_metrics_temporality_preference = ( environ.get( @@ -119,6 +129,13 @@ def _common_configuration( instrument_class_temporality.update(preferred_temporality or {}) + return instrument_class_temporality + + def _get_aggregation( + self, + preferred_aggregation: Dict[type, Aggregation], + ) -> Dict[type, Aggregation]: + otel_exporter_otlp_metrics_default_histogram_aggregation = environ.get( OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION, "explicit_bucket_histogram", @@ -128,7 +145,9 @@ def _common_configuration( "base2_exponential_bucket_histogram" ): - histogram_aggregation_type = ExponentialBucketHistogramAggregation + instrument_class_aggregation = { + Histogram: ExponentialBucketHistogramAggregation(), + } else: @@ -145,13 +164,13 @@ def _common_configuration( otel_exporter_otlp_metrics_default_histogram_aggregation, ) - histogram_aggregation_type = ExplicitBucketHistogramAggregation + instrument_class_aggregation = { + Histogram: ExplicitBucketHistogramAggregation(), + } - MetricExporter.__init__( - self, - preferred_temporality=instrument_class_temporality, - preferred_aggregation={Histogram: histogram_aggregation_type()}, - ) + instrument_class_aggregation.update(preferred_aggregation or {}) + + return instrument_class_aggregation def encode_metrics(data: MetricsData) -> ExportMetricsServiceRequest: diff --git a/exporter/opentelemetry-exporter-otlp-proto-common/tests/test_backoff.py b/exporter/opentelemetry-exporter-otlp-proto-common/tests/test_backoff.py new file mode 100644 index 00000000000..789a184ad04 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-proto-common/tests/test_backoff.py @@ -0,0 +1,46 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import TestCase + +from opentelemetry.exporter.otlp.proto.common._internal import ( + _create_exp_backoff_generator, +) + + +class TestBackoffGenerator(TestCase): + def test_exp_backoff_generator(self): + generator = _create_exp_backoff_generator() + self.assertEqual(next(generator), 1) + self.assertEqual(next(generator), 2) + self.assertEqual(next(generator), 4) + self.assertEqual(next(generator), 8) + self.assertEqual(next(generator), 16) + + def test_exp_backoff_generator_with_max(self): + generator = _create_exp_backoff_generator(max_value=4) + self.assertEqual(next(generator), 1) + self.assertEqual(next(generator), 2) + self.assertEqual(next(generator), 4) + self.assertEqual(next(generator), 4) + self.assertEqual(next(generator), 4) + + def test_exp_backoff_generator_with_odd_max(self): + # use a max_value that's not in the set + generator = _create_exp_backoff_generator(max_value=11) + self.assertEqual(next(generator), 1) + self.assertEqual(next(generator), 2) + self.assertEqual(next(generator), 4) + self.assertEqual(next(generator), 8) + self.assertEqual(next(generator), 11) diff --git a/exporter/opentelemetry-exporter-otlp-proto-grpc/pyproject.toml b/exporter/opentelemetry-exporter-otlp-proto-grpc/pyproject.toml index da67478b6ea..eeea4ea5174 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-grpc/pyproject.toml +++ b/exporter/opentelemetry-exporter-otlp-proto-grpc/pyproject.toml @@ -25,7 +25,6 @@ classifiers = [ ] dependencies = [ "Deprecated >= 1.2.6", - "backoff >= 1.8.0, < 3.0.0", "googleapis-common-protos ~= 1.52", "grpcio >= 1.0.0, < 2.0.0", "opentelemetry-api ~= 1.15", diff --git a/exporter/opentelemetry-exporter-otlp-proto-grpc/src/opentelemetry/exporter/otlp/proto/grpc/metric_exporter/__init__.py b/exporter/opentelemetry-exporter-otlp-proto-grpc/src/opentelemetry/exporter/otlp/proto/grpc/metric_exporter/__init__.py index 2560c5c3057..0ceca25c867 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-grpc/src/opentelemetry/exporter/otlp/proto/grpc/metric_exporter/__init__.py +++ b/exporter/opentelemetry-exporter-otlp-proto-grpc/src/opentelemetry/exporter/otlp/proto/grpc/metric_exporter/__init__.py @@ -127,7 +127,9 @@ def __init__( else compression ) - self._common_configuration(preferred_temporality) + self._common_configuration( + preferred_temporality, preferred_aggregation + ) OTLPExporterMixin.__init__( self, diff --git a/exporter/opentelemetry-exporter-otlp-proto-grpc/tests/test_otlp_metrics_exporter.py b/exporter/opentelemetry-exporter-otlp-proto-grpc/tests/test_otlp_metrics_exporter.py index 291e9457efd..95733b917bf 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-grpc/tests/test_otlp_metrics_exporter.py +++ b/exporter/opentelemetry-exporter-otlp-proto-grpc/tests/test_otlp_metrics_exporter.py @@ -968,6 +968,24 @@ def test_exponential_explicit_bucket_histogram(self): ExplicitBucketHistogramAggregation, ) + def test_preferred_aggregation_override(self): + + histogram_aggregation = ExplicitBucketHistogramAggregation( + boundaries=[0.05, 0.1, 0.5, 1, 5, 10], + ) + + exporter = OTLPMetricExporter( + preferred_aggregation={ + Histogram: histogram_aggregation, + }, + ) + + self.assertEqual( + # pylint: disable=protected-access + exporter._preferred_aggregation[Histogram], + histogram_aggregation, + ) + def _resource_metrics( index: int, scope_metrics: List[ScopeMetrics] diff --git a/exporter/opentelemetry-exporter-otlp-proto-grpc/tests/test_otlp_trace_exporter.py b/exporter/opentelemetry-exporter-otlp-proto-grpc/tests/test_otlp_trace_exporter.py index 5445ddf9262..bb17e35b7b7 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-grpc/tests/test_otlp_trace_exporter.py +++ b/exporter/opentelemetry-exporter-otlp-proto-grpc/tests/test_otlp_trace_exporter.py @@ -15,7 +15,6 @@ import os import threading import time -from collections import OrderedDict from concurrent.futures import ThreadPoolExecutor from logging import WARNING from unittest import TestCase @@ -28,7 +27,6 @@ from opentelemetry.attributes import BoundedAttributes from opentelemetry.exporter.otlp.proto.common._internal import ( _encode_key_value, - _is_backoff_v2, ) from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( OTLPSpanExporter, @@ -154,12 +152,12 @@ def setUp(self): "a", context=Mock( **{ - "trace_state": OrderedDict([("a", "b"), ("c", "d")]), + "trace_state": {"a": "b", "c": "d"}, "span_id": 10217189687419569865, "trace_id": 67545097771067222548457157018666467027, } ), - resource=SDKResource(OrderedDict([("a", 1), ("b", False)])), + resource=SDKResource({"a": 1, "b": False}), parent=Mock(**{"span_id": 12345}), attributes=BoundedAttributes(attributes={"a": 1, "b": True}), events=[event_mock], @@ -184,12 +182,12 @@ def setUp(self): "b", context=Mock( **{ - "trace_state": OrderedDict([("a", "b"), ("c", "d")]), + "trace_state": {"a": "b", "c": "d"}, "span_id": 10217189687419569865, "trace_id": 67545097771067222548457157018666467027, } ), - resource=SDKResource(OrderedDict([("a", 2), ("b", False)])), + resource=SDKResource({"a": 2, "b": False}), parent=Mock(**{"span_id": 12345}), instrumentation_scope=InstrumentationScope( name="name", version="version" @@ -200,12 +198,12 @@ def setUp(self): "c", context=Mock( **{ - "trace_state": OrderedDict([("a", "b"), ("c", "d")]), + "trace_state": {"a": "b", "c": "d"}, "span_id": 10217189687419569865, "trace_id": 67545097771067222548457157018666467027, } ), - resource=SDKResource(OrderedDict([("a", 1), ("b", False)])), + resource=SDKResource({"a": 1, "b": False}), parent=Mock(**{"span_id": 12345}), instrumentation_scope=InstrumentationScope( name="name2", version="version2" @@ -460,23 +458,6 @@ def test_otlp_headers(self, mock_ssl_channel, mock_secure): (("user-agent", "OTel-OTLP-Exporter-Python/" + __version__),), ) - @patch("opentelemetry.exporter.otlp.proto.common._internal.backoff") - @patch("opentelemetry.exporter.otlp.proto.grpc.exporter.sleep") - def test_handles_backoff_v2_api(self, mock_sleep, mock_backoff): - # In backoff ~= 2.0.0 the first value yielded from expo is None. - def generate_delays(*args, **kwargs): - if _is_backoff_v2: - yield None - yield 1 - - mock_backoff.expo.configure_mock(**{"side_effect": generate_delays}) - - add_TraceServiceServicer_to_server( - TraceServiceServicerUNAVAILABLE(), self.server - ) - self.exporter.export([self.span]) - mock_sleep.assert_called_once_with(1) - @patch( "opentelemetry.exporter.otlp.proto.grpc.exporter._create_exp_backoff_generator" ) @@ -982,7 +963,7 @@ def _create_span_with_status(status: SDKStatus): "a", context=Mock( **{ - "trace_state": OrderedDict([("a", "b"), ("c", "d")]), + "trace_state": {"a": "b", "c": "d"}, "span_id": 10217189687419569865, "trace_id": 67545097771067222548457157018666467027, } diff --git a/exporter/opentelemetry-exporter-otlp-proto-http/pyproject.toml b/exporter/opentelemetry-exporter-otlp-proto-http/pyproject.toml index 35c4ca24a3c..740f05c3cfe 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-http/pyproject.toml +++ b/exporter/opentelemetry-exporter-otlp-proto-http/pyproject.toml @@ -25,7 +25,6 @@ classifiers = [ ] dependencies = [ "Deprecated >= 1.2.6", - "backoff >= 1.10.0, < 3.0.0", "googleapis-common-protos ~= 1.52", "opentelemetry-api ~= 1.15", "opentelemetry-proto == 1.23.0.dev", diff --git a/exporter/opentelemetry-exporter-otlp-proto-http/src/opentelemetry/exporter/otlp/proto/http/metric_exporter/__init__.py b/exporter/opentelemetry-exporter-otlp-proto-http/src/opentelemetry/exporter/otlp/proto/http/metric_exporter/__init__.py index becdab257fe..6be74a37a06 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-http/src/opentelemetry/exporter/otlp/proto/http/metric_exporter/__init__.py +++ b/exporter/opentelemetry-exporter-otlp-proto-http/src/opentelemetry/exporter/otlp/proto/http/metric_exporter/__init__.py @@ -135,7 +135,9 @@ def __init__( {"Content-Encoding": self._compression.value} ) - self._common_configuration(preferred_temporality) + self._common_configuration( + preferred_temporality, preferred_aggregation + ) def _export(self, serialized_data: str): data = serialized_data diff --git a/exporter/opentelemetry-exporter-otlp-proto-http/tests/metrics/test_otlp_metrics_exporter.py b/exporter/opentelemetry-exporter-otlp-proto-http/tests/metrics/test_otlp_metrics_exporter.py index c06b5db3c22..674785056a5 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-http/tests/metrics/test_otlp_metrics_exporter.py +++ b/exporter/opentelemetry-exporter-otlp-proto-http/tests/metrics/test_otlp_metrics_exporter.py @@ -15,13 +15,12 @@ from logging import WARNING from os import environ from unittest import TestCase -from unittest.mock import MagicMock, Mock, patch +from unittest.mock import MagicMock, Mock, call, patch from requests import Session from requests.models import Response from responses import POST, activate, add -from opentelemetry.exporter.otlp.proto.common._internal import _is_backoff_v2 from opentelemetry.exporter.otlp.proto.common.metrics_encoder import ( encode_metrics, ) @@ -298,17 +297,8 @@ def test_serialization(self, mock_post): ) @activate - @patch("opentelemetry.exporter.otlp.proto.common._internal.backoff") @patch("opentelemetry.exporter.otlp.proto.http.metric_exporter.sleep") - def test_handles_backoff_v2_api(self, mock_sleep, mock_backoff): - # In backoff ~= 2.0.0 the first value yielded from expo is None. - def generate_delays(*args, **kwargs): - if _is_backoff_v2: - yield None - yield 1 - - mock_backoff.expo.configure_mock(**{"side_effect": generate_delays}) - + def test_exponential_backoff(self, mock_sleep): # return a retryable error add( POST, @@ -323,7 +313,9 @@ def generate_delays(*args, **kwargs): metrics_data = self.metrics["sum_int"] exporter.export(metrics_data) - mock_sleep.assert_called_once_with(1) + mock_sleep.assert_has_calls( + [call(1), call(2), call(4), call(8), call(16), call(32)] + ) def test_aggregation_temporality(self): @@ -487,3 +479,19 @@ def test_2xx_status_code(self, mock_otlp_metric_exporter): OTLPMetricExporter().export(MagicMock()), MetricExportResult.SUCCESS, ) + + def test_preferred_aggregation_override(self): + + histogram_aggregation = ExplicitBucketHistogramAggregation( + boundaries=[0.05, 0.1, 0.5, 1, 5, 10], + ) + + exporter = OTLPMetricExporter( + preferred_aggregation={ + Histogram: histogram_aggregation, + }, + ) + + self.assertEqual( + exporter._preferred_aggregation[Histogram], histogram_aggregation + ) diff --git a/exporter/opentelemetry-exporter-otlp-proto-http/tests/test_proto_log_exporter.py b/exporter/opentelemetry-exporter-otlp-proto-http/tests/test_proto_log_exporter.py index e601e5d00cb..6b6aafd465f 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-http/tests/test_proto_log_exporter.py +++ b/exporter/opentelemetry-exporter-otlp-proto-http/tests/test_proto_log_exporter.py @@ -16,13 +16,12 @@ import unittest from typing import List -from unittest.mock import MagicMock, Mock, patch +from unittest.mock import MagicMock, Mock, call, patch import requests import responses from opentelemetry._logs import SeverityNumber -from opentelemetry.exporter.otlp.proto.common._internal import _is_backoff_v2 from opentelemetry.exporter.otlp.proto.http import Compression from opentelemetry.exporter.otlp.proto.http._log_exporter import ( DEFAULT_COMPRESSION, @@ -169,17 +168,8 @@ def test_exporter_env(self): self.assertIsInstance(exporter._session, requests.Session) @responses.activate - @patch("opentelemetry.exporter.otlp.proto.common._internal.backoff") @patch("opentelemetry.exporter.otlp.proto.http._log_exporter.sleep") - def test_handles_backoff_v2_api(self, mock_sleep, mock_backoff): - # In backoff ~= 2.0.0 the first value yielded from expo is None. - def generate_delays(*args, **kwargs): - if _is_backoff_v2: - yield None - yield 1 - - mock_backoff.expo.configure_mock(**{"side_effect": generate_delays}) - + def test_exponential_backoff(self, mock_sleep): # return a retryable error responses.add( responses.POST, @@ -192,7 +182,9 @@ def generate_delays(*args, **kwargs): logs = self._get_sdk_log_data() exporter.export(logs) - mock_sleep.assert_called_once_with(1) + mock_sleep.assert_has_calls( + [call(1), call(2), call(4), call(8), call(16), call(32)] + ) @staticmethod def _get_sdk_log_data() -> List[LogData]: diff --git a/exporter/opentelemetry-exporter-otlp-proto-http/tests/test_proto_span_exporter.py b/exporter/opentelemetry-exporter-otlp-proto-http/tests/test_proto_span_exporter.py index eb5b375e40d..69874664c7a 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-http/tests/test_proto_span_exporter.py +++ b/exporter/opentelemetry-exporter-otlp-proto-http/tests/test_proto_span_exporter.py @@ -13,13 +13,11 @@ # limitations under the License. import unittest -from collections import OrderedDict -from unittest.mock import MagicMock, Mock, patch +from unittest.mock import MagicMock, Mock, call, patch import requests import responses -from opentelemetry.exporter.otlp.proto.common._internal import _is_backoff_v2 from opentelemetry.exporter.otlp.proto.http import Compression from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( DEFAULT_COMPRESSION, @@ -205,17 +203,8 @@ def test_headers_parse_from_env(self): # pylint: disable=no-self-use @responses.activate - @patch("opentelemetry.exporter.otlp.proto.common._internal.backoff") @patch("opentelemetry.exporter.otlp.proto.http.trace_exporter.sleep") - def test_handles_backoff_v2_api(self, mock_sleep, mock_backoff): - # In backoff ~= 2.0.0 the first value yielded from expo is None. - def generate_delays(*args, **kwargs): - if _is_backoff_v2: - yield None - yield 1 - - mock_backoff.expo.configure_mock(**{"side_effect": generate_delays}) - + def test_exponential_backoff(self, mock_sleep): # return a retryable error responses.add( responses.POST, @@ -231,7 +220,7 @@ def generate_delays(*args, **kwargs): "abc", context=Mock( **{ - "trace_state": OrderedDict([("a", "b"), ("c", "d")]), + "trace_state": {"a": "b", "c": "d"}, "span_id": 10217189687419569865, "trace_id": 67545097771067222548457157018666467027, } @@ -239,7 +228,9 @@ def generate_delays(*args, **kwargs): ) exporter.export([span]) - mock_sleep.assert_called_once_with(1) + mock_sleep.assert_has_calls( + [call(1), call(2), call(4), call(8), call(16), call(32)] + ) @patch.object(OTLPSpanExporter, "_export", return_value=Mock(ok=True)) def test_2xx_status_code(self, mock_otlp_metric_exporter): diff --git a/opentelemetry-api/src/opentelemetry/attributes/__init__.py b/opentelemetry-api/src/opentelemetry/attributes/__init__.py index 783d18163ec..89c879459f1 100644 --- a/opentelemetry-api/src/opentelemetry/attributes/__init__.py +++ b/opentelemetry-api/src/opentelemetry/attributes/__init__.py @@ -148,7 +148,8 @@ def __init__( self.maxlen = maxlen self.dropped = 0 self.max_value_len = max_value_len - self._dict = OrderedDict() # type: OrderedDict + # OrderedDict is not used until the maxlen is reached for efficiency. + self._dict = {} # type: dict | OrderedDict self._lock = threading.Lock() # type: threading.Lock if attributes: for key, value in attributes.items(): @@ -178,6 +179,8 @@ def __setitem__(self, key, value): elif ( self.maxlen is not None and len(self._dict) == self.maxlen ): + if not isinstance(self._dict, OrderedDict): + self._dict = OrderedDict(self._dict) self._dict.popitem(last=False) self.dropped += 1 diff --git a/opentelemetry-api/src/opentelemetry/trace/span.py b/opentelemetry-api/src/opentelemetry/trace/span.py index 805b2b06b18..327d85fef48 100644 --- a/opentelemetry-api/src/opentelemetry/trace/span.py +++ b/opentelemetry-api/src/opentelemetry/trace/span.py @@ -3,7 +3,6 @@ import re import types as python_types import typing -from collections import OrderedDict from opentelemetry.trace.status import Status, StatusCode from opentelemetry.util import types @@ -218,7 +217,7 @@ def __init__( typing.Sequence[typing.Tuple[str, str]] ] = None, ) -> None: - self._dict = OrderedDict() # type: OrderedDict[str, str] + self._dict = {} # type: dict[str, str] if entries is None: return if len(entries) > _TRACECONTEXT_MAXIMUM_TRACESTATE_KEYS: @@ -310,9 +309,8 @@ def update(self, key: str, value: str) -> "TraceState": ) return self prev_state = self._dict.copy() - prev_state[key] = value - prev_state.move_to_end(key, last=False) - new_state = list(prev_state.items()) + prev_state.pop(key, None) + new_state = [(key, value), *prev_state.items()] return TraceState(new_state) def delete(self, key: str) -> "TraceState": @@ -362,7 +360,7 @@ def from_header(cls, header_list: typing.List[str]) -> "TraceState": If the number of keys is beyond the maximum, all values will be discarded and an empty tracestate will be returned. """ - pairs = OrderedDict() # type: OrderedDict[str, str] + pairs = {} # type: dict[str, str] for header in header_list: members: typing.List[str] = re.split(_delimiter_pattern, header) for member in members: diff --git a/opentelemetry-api/tests/attributes/test_attributes.py b/opentelemetry-api/tests/attributes/test_attributes.py index 121dec3d251..ad2f741fb1f 100644 --- a/opentelemetry-api/tests/attributes/test_attributes.py +++ b/opentelemetry-api/tests/attributes/test_attributes.py @@ -14,7 +14,6 @@ # type: ignore -import collections import unittest from typing import MutableSequence @@ -90,14 +89,12 @@ def test_sequence_attr_decode(self): class TestBoundedAttributes(unittest.TestCase): - base = collections.OrderedDict( - [ - ("name", "Firulais"), - ("age", 7), - ("weight", 13), - ("vaccinated", True), - ] - ) + base = { + "name": "Firulais", + "age": 7, + "weight": 13, + "vaccinated": True, + } def test_negative_maxlen(self): with self.assertRaises(ValueError): @@ -105,7 +102,7 @@ def test_negative_maxlen(self): def test_from_map(self): dic_len = len(self.base) - base_copy = collections.OrderedDict(self.base) + base_copy = self.base.copy() bdict = BoundedAttributes(dic_len, base_copy) self.assertEqual(len(bdict), dic_len) diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py index 0e85b464c33..783a99a7d51 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py @@ -466,6 +466,7 @@ def _get_attributes(self, record: logging.LogRecord) -> Attributes: def _translate(self, record: logging.LogRecord) -> LogRecord: timestamp = int(record.created * 1e9) + observered_timestamp = time_ns() span_context = get_current_span().get_span_context() attributes = self._get_attributes(record) severity_number = std_to_otel(record.levelno) @@ -479,6 +480,7 @@ def _translate(self, record: logging.LogRecord) -> LogRecord: return LogRecord( timestamp=timestamp, + observed_timestamp=observered_timestamp, trace_id=span_context.trace_id, span_id=span_context.span_id, trace_flags=span_context.trace_flags, @@ -500,9 +502,10 @@ def emit(self, record: logging.LogRecord) -> None: def flush(self) -> None: """ - Flushes the logging output. + Flushes the logging output. Skip flushing if logger is NoOp. """ - self._logger_provider.force_flush() + if not isinstance(self._logger, NoOpLogger): + self._logger_provider.force_flush() class Logger(APILogger): diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py index 6dae70b2f6b..344838ba186 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py @@ -1042,8 +1042,8 @@ def start_as_current_span( end_on_exit=end_on_exit, record_exception=record_exception, set_status_on_exception=set_status_on_exception, - ) as span_context: - yield span_context + ) as span: + yield span def start_span( # pylint: disable=too-many-locals self, diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/util/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/util/__init__.py index e1857d8e62d..37ca62b017c 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/util/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/util/__init__.py @@ -14,7 +14,7 @@ import datetime import threading -from collections import OrderedDict, deque +from collections import deque from collections.abc import MutableMapping, Sequence from typing import Optional @@ -107,7 +107,7 @@ def __init__(self, maxlen: Optional[int]): raise ValueError self.maxlen = maxlen self.dropped = 0 - self._dict = OrderedDict() # type: OrderedDict + self._dict = {} # type: dict self._lock = threading.Lock() # type: threading.Lock def __repr__(self): @@ -143,7 +143,7 @@ def __len__(self): @classmethod def from_map(cls, maxlen, mapping): - mapping = OrderedDict(mapping) + mapping = dict(mapping) bounded_dict = cls(maxlen) for key, value in mapping.items(): bounded_dict[key] = value diff --git a/opentelemetry-sdk/tests/logs/test_handler.py b/opentelemetry-sdk/tests/logs/test_handler.py index 2bd82efd464..7a32a3ed75c 100644 --- a/opentelemetry-sdk/tests/logs/test_handler.py +++ b/opentelemetry-sdk/tests/logs/test_handler.py @@ -81,6 +81,19 @@ def test_log_record_emit_noop(self): logger.warning("Warning message") handler_mock._translate.assert_not_called() + def test_log_flush_noop(self): + + no_op_logger_provider = NoOpLoggerProvider() + no_op_logger_provider.force_flush = Mock() + + logger = get_logger(logger_provider=no_op_logger_provider) + + with self.assertLogs(level=logging.WARNING): + logger.warning("Warning message") + + logger.handlers[0].flush() + no_op_logger_provider.force_flush.assert_not_called() + def test_log_record_no_span_context(self): emitter_provider_mock = Mock(spec=LoggerProvider) emitter_mock = APIGetLogger( @@ -100,6 +113,20 @@ def test_log_record_no_span_context(self): log_record.trace_flags, INVALID_SPAN_CONTEXT.trace_flags ) + def test_log_record_observed_timestamp(self): + emitter_provider_mock = Mock(spec=LoggerProvider) + emitter_mock = APIGetLogger( + __name__, logger_provider=emitter_provider_mock + ) + logger = get_logger(logger_provider=emitter_provider_mock) + # Assert emit gets called for warning message + with self.assertLogs(level=logging.WARNING): + logger.warning("Warning message") + args, _ = emitter_mock.emit.call_args_list[0] + log_record = args[0] + + self.assertIsNotNone(log_record.observed_timestamp) + def test_log_record_user_attributes(self): """Attributes can be injected into logs by adding them to the LogRecord""" emitter_provider_mock = Mock(spec=LoggerProvider) diff --git a/tox.ini b/tox.ini index 035f8269e37..1bd0c98226e 100644 --- a/tox.ini +++ b/tox.ini @@ -20,6 +20,7 @@ envlist = ; docs/getting-started py3{8,9,10,11}-opentelemetry-getting-started + pypy3-opentelemetry-getting-started py3{8,9,10,11}-opentelemetry-opentracing-shim pypy3-opentelemetry-opentracing-shim @@ -31,6 +32,7 @@ envlist = ; exporter-opencensus intentionally excluded from pypy3 py3{8,9,10,11}-proto{3,4}-opentelemetry-exporter-otlp-proto-common + pypy3-proto{3,4}-opentelemetry-exporter-otlp-proto-common ; opentelemetry-exporter-otlp py3{8,9,10,11}-opentelemetry-exporter-otlp-combined @@ -90,8 +92,8 @@ deps = setenv = ; override CONTRIB_REPO_SHA via env variable when testing other branches/commits than main ; i.e: CONTRIB_REPO_SHA=dde62cebffe519c35875af6d06fae053b3be65ec tox -e - CONTRIB_REPO_SHA={env:CONTRIB_REPO_SHA:"main"} - CONTRIB_REPO="git+https://github.com/open-telemetry/opentelemetry-python-contrib.git@{env:CONTRIB_REPO_SHA}" + CONTRIB_REPO_SHA={env:CONTRIB_REPO_SHA:main} + CONTRIB_REPO=git+https://github.com/open-telemetry/opentelemetry-python-contrib.git@{env:CONTRIB_REPO_SHA} mypy: MYPYPATH={toxinidir}/opentelemetry-api/src/:{toxinidir}/tests/opentelemetry-test-utils/src/ changedir = @@ -127,11 +129,11 @@ commands_pre = protobuf: pip install {toxinidir}/opentelemetry-proto getting-started: pip install -r requirements.txt - getting-started: pip install -e "{env:CONTRIB_REPO}#egg=opentelemetry-util-http&subdirectory=util/opentelemetry-util-http" - getting-started: pip install -e "{env:CONTRIB_REPO}#egg=opentelemetry-instrumentation&subdirectory=opentelemetry-instrumentation" - getting-started: pip install -e "{env:CONTRIB_REPO}#egg=opentelemetry-instrumentation-requests&subdirectory=instrumentation/opentelemetry-instrumentation-requests" - getting-started: pip install -e "{env:CONTRIB_REPO}#egg=opentelemetry-instrumentation-wsgi&subdirectory=instrumentation/opentelemetry-instrumentation-wsgi" - getting-started: pip install -e "{env:CONTRIB_REPO}#egg=opentelemetry-instrumentation-flask&subdirectory=instrumentation/opentelemetry-instrumentation-flask" + getting-started: pip install -e {env:CONTRIB_REPO}\#egg=opentelemetry-util-http&subdirectory=util/opentelemetry-util-http + getting-started: pip install -e {env:CONTRIB_REPO}\#egg=opentelemetry-instrumentation&subdirectory=opentelemetry-instrumentation + getting-started: pip install -e {env:CONTRIB_REPO}\#egg=opentelemetry-instrumentation-requests&subdirectory=instrumentation/opentelemetry-instrumentation-requests + getting-started: pip install -e {env:CONTRIB_REPO}\#egg=opentelemetry-instrumentation-wsgi&subdirectory=instrumentation/opentelemetry-instrumentation-wsgi + getting-started: pip install -e {env:CONTRIB_REPO}\#egg=opentelemetry-instrumentation-flask&subdirectory=instrumentation/opentelemetry-instrumentation-flask opencensus: pip install {toxinidir}/exporter/opentelemetry-exporter-opencensus @@ -255,14 +257,17 @@ deps = requests~=2.7 markupsafe~=2.1 +allowlist_externals = + {toxinidir}/scripts/tracecontext-integration-test.sh + commands_pre = pip install -e {toxinidir}/opentelemetry-api \ -e {toxinidir}/opentelemetry-semantic-conventions \ -e {toxinidir}/opentelemetry-sdk \ - -e "{env:CONTRIB_REPO}#egg=opentelemetry-util-http&subdirectory=util/opentelemetry-util-http" \ - -e "{env:CONTRIB_REPO}#egg=opentelemetry-instrumentation&subdirectory=opentelemetry-instrumentation" \ - -e "{env:CONTRIB_REPO}#egg=opentelemetry-instrumentation-requests&subdirectory=instrumentation/opentelemetry-instrumentation-requests" \ - -e "{env:CONTRIB_REPO}#egg=opentelemetry-instrumentation-wsgi&subdirectory=instrumentation/opentelemetry-instrumentation-wsgi" + -e {env:CONTRIB_REPO}\#egg=opentelemetry-util-http&subdirectory=util/opentelemetry-util-http \ + -e {env:CONTRIB_REPO}\#egg=opentelemetry-instrumentation&subdirectory=opentelemetry-instrumentation \ + -e {env:CONTRIB_REPO}\#egg=opentelemetry-instrumentation-requests&subdirectory=instrumentation/opentelemetry-instrumentation-requests \ + -e {env:CONTRIB_REPO}\#egg=opentelemetry-instrumentation-wsgi&subdirectory=instrumentation/opentelemetry-instrumentation-wsgi commands = {toxinidir}/scripts/tracecontext-integration-test.sh