Add annotation to slower tests so devs can skip them

CONTRIBUTING.md

@@ -97,6 +97,13 @@ See
 [`tox.ini`](https://github.com/open-telemetry/opentelemetry-python/blob/main/tox.ini)
 for more detail on available tox commands.
 
+#### Tests
+
+For developer convenience, test methods that take longer than roughly a full second may be marked with the annotation
+`@pytest.mark.slow`. If you add a test that takes longer than a second, you are encouraged to add this annotation. To
+then run tests while filtering out any marked as `slow`, you can add `-m 'not slow'` to `tox` or `pytest`, as in
+`tox -- -m 'not slow'` or `pytest -m 'not slow'`. However, by default, all tests are run.
+
 #### Contrib repo
 
 Some of the `tox` targets install packages from the [OpenTelemetry Python Contrib Repository](https://github.com/open-telemetry/opentelemetry-python.git) via

exporter/opentelemetry-exporter-otlp-proto-grpc/tests/performance/benchmarks/test_benchmark_trace_exporter.py

@@ -14,6 +14,8 @@
 
 from unittest.mock import patch
 
+import pytest
+
 from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import (
     OTLPSpanExporter,
 )
@@ -38,6 +40,7 @@ def __init__(self, channel):
         self.Export = lambda *args, **kwargs: None
 
 
+@pytest.mark.slow
 @patch(
     "opentelemetry.exporter.otlp.proto.grpc.trace_exporter.OTLPSpanExporter._stub",
     new=MockTraceServiceStub,
@@ -59,6 +62,7 @@ def create_spans_to_be_exported():
     benchmark(create_spans_to_be_exported)
 
 
+@pytest.mark.slow
 @patch(
     "opentelemetry.exporter.otlp.proto.grpc.trace_exporter.OTLPSpanExporter._stub",
     new=MockTraceServiceStub,

exporter/opentelemetry-exporter-otlp-proto-grpc/tests/test_otlp_exporter_mixin.py

@@ -20,6 +20,7 @@
 from unittest import TestCase
 from unittest.mock import Mock, patch
 
+import pytest
 from google.protobuf.duration_pb2 import Duration
 from google.rpc.error_details_pb2 import RetryInfo
 from grpc import Compression
@@ -151,6 +152,7 @@ def _exporting(self) -> str:
                 "Exporter already shutdown, ignoring batch",
             )
 
+    @pytest.mark.slow
     def test_shutdown_wait_last_export(self):
         result_mock = Mock()
         rpc_error = RpcError()

exporter/opentelemetry-exporter-otlp-proto-grpc/tests/test_otlp_metrics_exporter.py

@@ -24,6 +24,7 @@
 from unittest import TestCase
 from unittest.mock import patch
 
+import pytest
 from google.protobuf.duration_pb2 import Duration
 from google.rpc.error_details_pb2 import RetryInfo
 from grpc import ChannelCredentials, Compression, StatusCode, server
@@ -754,6 +755,7 @@ def test_shutdown(self):
             )
         self.exporter = OTLPMetricExporter()
 
+    @pytest.mark.slow
     def test_shutdown_wait_last_export(self):
         add_MetricsServiceServicer_to_server(
             MetricsServiceServicerUNAVAILABLEDelay(), self.server

exporter/opentelemetry-exporter-otlp-proto-grpc/tests/test_otlp_trace_exporter.py

@@ -21,6 +21,7 @@
 from unittest import TestCase
 from unittest.mock import Mock, PropertyMock, patch
 
+import pytest
 from google.protobuf.duration_pb2 import Duration
 from google.rpc.error_details_pb2 import RetryInfo
 from grpc import ChannelCredentials, Compression, StatusCode, server
@@ -893,6 +894,7 @@ def test_translate_key_values(self):
         # self.assertEqual(kvlist_value.values[0].key, "asd")
         # self.assertEqual(kvlist_value.values[0].value.string_value, "123")
 
+    @pytest.mark.slow
     def test_dropped_values(self):
         span = get_span_with_dropped_attributes_events_links()
         # pylint:disable=protected-access
@@ -952,6 +954,7 @@ def test_shutdown(self):
                 "Exporter already shutdown, ignoring batch",
             )
 
+    @pytest.mark.slow
     def test_shutdown_wait_last_export(self):
         add_TraceServiceServicer_to_server(
             TraceServiceServicerUNAVAILABLEDelay(), self.server

exporter/opentelemetry-exporter-zipkin-proto-http/tests/encoder/test_v2_protobuf.py

@@ -14,6 +14,8 @@
 import ipaddress
 import json
 
+import pytest
+
 from opentelemetry.exporter.zipkin.encoder import (
     _SCOPE_NAME_KEY,
     _SCOPE_VERSION_KEY,
@@ -249,6 +251,7 @@ def _test_encode_max_tag_length(self, max_tag_value_length: int):
 
         self.assertEqual(actual_output, expected_output)
 
+    @pytest.mark.slow
     def test_dropped_span_attributes(self):
         otel_span = get_span_with_dropped_attributes_events_links()
         # pylint: disable=no-member

opentelemetry-api/tests/trace/test_globals.py

@@ -1,6 +1,8 @@
 import unittest
 from unittest.mock import Mock, patch
 
+import pytest
+
 from opentelemetry import context, trace
 from opentelemetry.test.concurrency_test import ConcurrencyTestBase, MockFunc
 from opentelemetry.test.globals_test import TraceGlobalsTest
@@ -40,6 +42,8 @@ def test_get_tracer(mock_tracer_provider):  # type: ignore
 
 
 class TestGlobalsConcurrency(TraceGlobalsTest, ConcurrencyTestBase):
+
+    @pytest.mark.slow
     @patch("opentelemetry.trace.logger")
     def test_set_tracer_provider_many_threads(self, mock_logger) -> None:  # type: ignore
         mock_logger.warning = MockFunc()

opentelemetry-sdk/tests/logs/test_export.py

@@ -21,6 +21,8 @@
 from concurrent.futures import ThreadPoolExecutor
 from unittest.mock import Mock, patch
 
+import pytest
+
 from opentelemetry._logs import SeverityNumber
 from opentelemetry.sdk import trace
 from opentelemetry.sdk._logs import (
@@ -444,6 +446,7 @@ def bulk_log_and_flush(num_logs):
         finished_logs = exporter.get_finished_logs()
         self.assertEqual(len(finished_logs), 2415)
 
+    @pytest.mark.slow
     @unittest.skipUnless(
         hasattr(os, "fork"),
         "needs *nix",

opentelemetry-sdk/tests/metrics/exponential_histogram/test_exponential_bucket_histogram_aggregation.py

@@ -14,6 +14,8 @@
 
 from itertools import permutations
 from logging import WARNING
+
+import pytest
 from math import ldexp
 from sys import float_info
 from types import MethodType
@@ -270,6 +272,7 @@ def test_permutations(self):
                     expected["at_1"],
                 )
 
+    @pytest.mark.slow
     def test_ascending_sequence(self):
 
         for max_size in [3, 4, 6, 9]:

opentelemetry-sdk/tests/metrics/integration_test/test_time_align.py

@@ -16,6 +16,7 @@
 from time import sleep
 from unittest import TestCase
 
+import pytest
 from pytest import mark
 
 from opentelemetry.sdk.metrics import Counter, MeterProvider
@@ -26,6 +27,8 @@
 
 
 class TestTimeAlign(TestCase):
+
+    @pytest.mark.slow
     def test_time_align_cumulative(self):
         reader = InMemoryMetricReader()
         meter_provider = MeterProvider(metric_readers=[reader])

opentelemetry-sdk/tests/metrics/test_measurement_consumer.py

@@ -17,6 +17,8 @@
 from unittest import TestCase
 from unittest.mock import MagicMock, Mock, patch
 
+import pytest
+
 from opentelemetry.sdk.metrics._internal.measurement_consumer import (
     MeasurementConsumer,
     SynchronousMeasurementConsumer,
@@ -118,6 +120,7 @@ def test_collect_calls_async_instruments(self, MockMetricReaderStorage):
             len(reader_storage_mock.consume_measurement.mock_calls), 5
         )
 
+    @pytest.mark.slow
     def test_collect_timeout(self, MockMetricReaderStorage):
         reader_mock = Mock()
         reader_storage_mock = Mock()
@@ -144,6 +147,7 @@ def sleep_1(*args, **kwargs):
             "Timed out while executing callback", error.exception.args[0]
         )
 
+    @pytest.mark.slow
     @patch(
         "opentelemetry.sdk.metrics._internal."
         "measurement_consumer.CallbackOptions"

opentelemetry-sdk/tests/metrics/test_metric_reader_storage.py

@@ -15,6 +15,8 @@
 from logging import WARNING
 from unittest.mock import MagicMock, Mock, patch
 
+import pytest
+
 from opentelemetry.sdk.metrics._internal.aggregation import (
     _LastValueAggregation,
 )
@@ -223,6 +225,7 @@ def test_forwards_calls_to_view_instrument_match(
             all_metrics[5],
         )
 
+    @pytest.mark.slow
     @patch(
         "opentelemetry.sdk.metrics._internal."
         "metric_reader_storage._ViewInstrumentMatch"

opentelemetry-sdk/tests/metrics/test_metrics.py

@@ -19,6 +19,8 @@
 from unittest import TestCase
 from unittest.mock import MagicMock, Mock, patch
 
+import pytest
+
 from opentelemetry.metrics import NoOpMeter
 from opentelemetry.sdk.metrics import (
     Counter,
@@ -471,6 +473,8 @@ def force_flush(self, timeout_millis: float = 10_000) -> bool:
 
 
 class TestDuplicateInstrumentAggregateData(TestCase):
+
+    @pytest.mark.slow
     def test_duplicate_instrument_aggregate_data(self):
 
         exporter = InMemoryMetricExporter()

opentelemetry-sdk/tests/performance/benchmarks/metrics/test_benchmark_metrics.py

@@ -36,6 +36,7 @@
 udcounter = meter_cumulative.create_up_down_counter("test_udcounter")
 
 
+@pytest.mark.slow
 @pytest.mark.parametrize(
     ("num_labels", "temporality"),
     [
@@ -65,6 +66,7 @@ def benchmark_counter_add():
     benchmark(benchmark_counter_add)
 
 
+@pytest.mark.slow
 @pytest.mark.parametrize("num_labels", [0, 1, 3, 5, 10])
 def test_up_down_counter_add(benchmark, num_labels):
     labels = {}

opentelemetry-sdk/tests/performance/benchmarks/metrics/test_benchmark_metrics_histogram,.py

@@ -66,6 +66,7 @@ def _generate_bounds(bound_count):
 hist1000 = meter.create_histogram("test_histogram_1000_bound")
 
 
+@pytest.mark.slow
 @pytest.mark.parametrize("num_labels", [0, 1, 3, 5, 7])
 def test_histogram_record(benchmark, num_labels):
     labels = {}
@@ -78,6 +79,7 @@ def benchmark_histogram_record():
     benchmark(benchmark_histogram_record)
 
 
+@pytest.mark.slow
 @pytest.mark.parametrize("num_labels", [0, 1, 3, 5, 7])
 def test_histogram_record_10(benchmark, num_labels):
     labels = {}
@@ -90,6 +92,7 @@ def benchmark_histogram_record_10():
     benchmark(benchmark_histogram_record_10)
 
 
+@pytest.mark.slow
 @pytest.mark.parametrize("num_labels", [0, 1, 3, 5, 7])
 def test_histogram_record_49(benchmark, num_labels):
     labels = {}
@@ -102,6 +105,7 @@ def benchmark_histogram_record_49():
     benchmark(benchmark_histogram_record_49)
 
 
+@pytest.mark.slow
 @pytest.mark.parametrize("num_labels", [0, 1, 3, 5, 7])
 def test_histogram_record_50(benchmark, num_labels):
     labels = {}
@@ -114,6 +118,7 @@ def benchmark_histogram_record_50():
     benchmark(benchmark_histogram_record_50)
 
 
+@pytest.mark.slow
 @pytest.mark.parametrize("num_labels", [0, 1, 3, 5, 7])
 def test_histogram_record_1000(benchmark, num_labels):
     labels = {}

opentelemetry-sdk/tests/performance/benchmarks/trace/test_benchmark_trace.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import pytest
 
 import opentelemetry.sdk.trace as trace
 from opentelemetry.sdk.resources import Resource
@@ -28,6 +29,7 @@
 ).get_tracer("sdk_tracer_provider")
 
 
+@pytest.mark.slow
 def test_simple_start_span(benchmark):
     def benchmark_start_as_current_span():
         span = tracer.start_span(
@@ -40,6 +42,7 @@ def benchmark_start_as_current_span():
     benchmark(benchmark_start_as_current_span)
 
 
+@pytest.mark.slow
 def test_simple_start_as_current_span(benchmark):
     def benchmark_start_as_current_span():
         with tracer.start_as_current_span(

opentelemetry-sdk/tests/trace/export/test_export.py

@@ -22,6 +22,7 @@
 from platform import python_implementation, system
 from unittest import mock
 
+import pytest
 from pytest import mark
 
 from opentelemetry import trace as trace_api
@@ -287,6 +288,7 @@ def test_flush_empty(self):
 
         self.assertTrue(span_processor.force_flush())
 
+    @pytest.mark.slow
     def test_flush_from_multiple_threads(self):
         num_threads = 50
         num_spans = 10
@@ -330,6 +332,7 @@ def test_flush_timeout(self):
             self.assertFalse(span_processor.force_flush(100))
         span_processor.shutdown()
 
+    @pytest.mark.slow
     def test_batch_span_processor_lossless(self):
         """Test that no spans are lost when sending max_queue_size spans"""
         spans_names_list = []
@@ -349,6 +352,7 @@ def test_batch_span_processor_lossless(self):
         self.assertEqual(len(spans_names_list), 512)
         span_processor.shutdown()
 
+    @pytest.mark.slow
     def test_batch_span_processor_many_spans(self):
         """Test that no spans are lost when sending many spans"""
         spans_names_list = []
@@ -404,6 +408,7 @@ def _check_fork_trace(self, exporter, expected):
         for span in spans:
             self.assertIn(span.name, expected)
 
+    @pytest.mark.slow
     @unittest.skipUnless(
         hasattr(os, "fork"),
         "needs *nix",

propagator/opentelemetry-propagator-b3/tests/performance/benchmarks/trace/propagation/test_benchmark_b3_format.py

@@ -11,13 +11,15 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import pytest
 
 import opentelemetry.propagators.b3 as b3_format
 import opentelemetry.sdk.trace as trace
 
 FORMAT = b3_format.B3Format()
 
 
+@pytest.mark.slow
 def test_extract_single_header(benchmark):
     benchmark(
         FORMAT.extract,

pyproject.toml

@@ -17,3 +17,6 @@ exclude = '''
 [tool.pytest.ini_options]
 addopts = "-rs -v"
 log_cli = true
+markers = [
+    "slow: marks tests as slow (run only fast tests with '-m \"not slow\"')"
+]