feat(otel): Autoinstrumentation skeleton (#3143)

Expand the POTel PoC's autoinstrumentation capabilities.
This change allows us to:
- install and enable all available instrumentations by default
- further configure instrumentations that accept optional arguments

.github/workflows/test-integrations-miscellaneous.yml

@@ -50,6 +50,10 @@ jobs:
         run: |
           set -x # print commands that are executed
           ./scripts/runtox.sh "py${{ matrix.python-version }}-opentelemetry-latest"
+      - name: Test potel latest
+        run: |
+          set -x # print commands that are executed
+          ./scripts/runtox.sh "py${{ matrix.python-version }}-potel-latest"
       - name: Test pure_eval latest
         run: |
           set -x # print commands that are executed
@@ -81,7 +85,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.6","3.7","3.8","3.9","3.11","3.12"]
+        python-version: ["3.6","3.7","3.8","3.9","3.10","3.11","3.12"]
         # python3.6 reached EOL and is no longer being supported on
         # new versions of hosted runners on Github Actions
         # ubuntu-20.04 is the last version that supported python3.6
@@ -106,6 +110,10 @@ jobs:
         run: |
           set -x # print commands that are executed
           ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-opentelemetry"
+      - name: Test potel pinned
+        run: |
+          set -x # print commands that are executed
+          ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-potel"
       - name: Test pure_eval pinned
         run: |
           set -x # print commands that are executed

scripts/split-tox-gh-actions/split-tox-gh-actions.py

@@ -124,6 +124,7 @@
     "Miscellaneous": [
         "loguru",
         "opentelemetry",
+        "potel",
         "pure_eval",
         "trytond",
     ],

sentry_sdk/client.py

@@ -358,9 +358,13 @@ def _capture_envelope(envelope):
                     "[OTel] Enabling experimental OTel-powered performance monitoring."
                 )
                 self.options["instrumenter"] = INSTRUMENTER.OTEL
-                _DEFAULT_INTEGRATIONS.append(
-                    "sentry_sdk.integrations.opentelemetry.integration.OpenTelemetryIntegration",
-                )
+                if (
+                    "sentry_sdk.integrations.opentelemetry.integration.OpenTelemetryIntegration"
+                    not in _DEFAULT_INTEGRATIONS
+                ):
+                    _DEFAULT_INTEGRATIONS.append(
+                        "sentry_sdk.integrations.opentelemetry.integration.OpenTelemetryIntegration",
+                    )
 
             self.integrations = setup_integrations(
                 self.options["integrations"],

sentry_sdk/integrations/opentelemetry/distro.py

@@ -0,0 +1,66 @@
+"""
+IMPORTANT: The contents of this file are part of a proof of concept and as such
+are experimental and not suitable for production use. They may be changed or
+removed at any time without prior notice.
+"""
+
+from sentry_sdk.integrations import DidNotEnable
+from sentry_sdk.integrations.opentelemetry.propagator import SentryPropagator
+from sentry_sdk.integrations.opentelemetry.span_processor import SentrySpanProcessor
+from sentry_sdk.utils import logger
+from sentry_sdk._types import TYPE_CHECKING
+
+try:
+    from opentelemetry import trace  # type: ignore
+    from opentelemetry.instrumentation.distro import BaseDistro  # type: ignore
+    from opentelemetry.propagate import set_global_textmap  # type: ignore
+    from opentelemetry.sdk.trace import TracerProvider  # type: ignore
+except ImportError:
+    raise DidNotEnable("opentelemetry not installed")
+
+try:
+    from opentelemetry.instrumentation.django import DjangoInstrumentor  # type: ignore
+except ImportError:
+    DjangoInstrumentor = None
+
+try:
+    from opentelemetry.instrumentation.flask import FlaskInstrumentor  # type: ignore
+except ImportError:
+    FlaskInstrumentor = None
+
+if TYPE_CHECKING:
+    # XXX pkg_resources is deprecated, there's a PR to switch to importlib:
+    # https://github.com/open-telemetry/opentelemetry-python-contrib/pull/2181
+    # we should align this when the PR gets merged
+    from pkg_resources import EntryPoint
+    from typing import Any
+
+
+CONFIGURABLE_INSTRUMENTATIONS = {
+    DjangoInstrumentor: {"is_sql_commentor_enabled": True},
+    FlaskInstrumentor: {"enable_commenter": True},
+}
+
+
+class _SentryDistro(BaseDistro):  # type: ignore[misc]
+    def _configure(self, **kwargs):
+        # type: (Any) -> None
+        provider = TracerProvider()
+        provider.add_span_processor(SentrySpanProcessor())
+        trace.set_tracer_provider(provider)
+        set_global_textmap(SentryPropagator())
+
+    def load_instrumentor(self, entry_point, **kwargs):
+        # type: (EntryPoint, Any) -> None
+        instrumentor = entry_point.load()
+
+        if instrumentor in CONFIGURABLE_INSTRUMENTATIONS:
+            for key, value in CONFIGURABLE_INSTRUMENTATIONS[instrumentor].items():
+                kwargs[key] = value
+
+        instrumentor().instrument(**kwargs)
+        logger.debug(
+            "[OTel] %s instrumented (%s)",
+            entry_point.name,
+            ", ".join([f"{k}: {v}" for k, v in kwargs.items()]),
+        )

sentry_sdk/integrations/opentelemetry/integration.py

@@ -8,19 +8,14 @@
 from importlib import import_module
 
 from sentry_sdk.integrations import DidNotEnable, Integration
-from sentry_sdk.integrations.opentelemetry.span_processor import SentrySpanProcessor
-from sentry_sdk.integrations.opentelemetry.propagator import SentryPropagator
+from sentry_sdk.integrations.opentelemetry.distro import _SentryDistro
 from sentry_sdk.utils import logger, _get_installed_modules
 from sentry_sdk._types import TYPE_CHECKING
 
 try:
-    from opentelemetry import trace  # type: ignore
     from opentelemetry.instrumentation.auto_instrumentation._load import (  # type: ignore
-        _load_distro,
         _load_instrumentors,
     )
-    from opentelemetry.propagate import set_global_textmap  # type: ignore
-    from opentelemetry.sdk.trace import TracerProvider  # type: ignore
 except ImportError:
     raise DidNotEnable("opentelemetry not installed")
 
@@ -34,6 +29,7 @@
     # instrumentation took place.
     "fastapi": "fastapi.FastAPI",
     "flask": "flask.Flask",
+    # XXX Add a mapping for all instrumentors that patch by replacing a class
 }
 
 
@@ -51,12 +47,21 @@ def setup_once():
         original_classes = _record_unpatched_classes()
 
         try:
-            distro = _load_distro()
+            distro = _SentryDistro()
             distro.configure()
+            # XXX This does some initial checks before loading instrumentations
+            # (checks OTEL_PYTHON_DISABLED_INSTRUMENTATIONS, checks version
+            # compat). If we don't want this in the future, we can implement our
+            # own _load_instrumentors (it anyway just iterates over
+            # opentelemetry_instrumentor entry points).
             _load_instrumentors(distro)
         except Exception:
             logger.exception("[OTel] Failed to auto-initialize OpenTelemetry")
 
+        # XXX: Consider whether this is ok to keep and make default.
+        # The alternative is asking folks to follow specific import order for
+        # some integrations (sentry_sdk.init before you even import Flask, for
+        # instance).
         try:
             _patch_remaining_classes(original_classes)
         except Exception:
@@ -65,8 +70,6 @@ def setup_once():
                 "You might have to make sure sentry_sdk.init() is called before importing anything else."
             )
 
-        _setup_sentry_tracing()
-
         logger.debug("[OTel] Finished setting up OpenTelemetry integration")
 
 
@@ -161,14 +164,3 @@ def _import_by_path(path):
     # type: (str) -> type
     parts = path.rsplit(".", maxsplit=1)
     return getattr(import_module(parts[0]), parts[-1])
-
-
-def _setup_sentry_tracing():
-    # type: () -> None
-    provider = TracerProvider()
-
-    provider.add_span_processor(SentrySpanProcessor())
-
-    trace.set_tracer_provider(provider)
-
-    set_global_textmap(SentryPropagator())

setup.py

@@ -66,14 +66,59 @@ def get_file_text(file_name):
         "openai": ["openai>=1.0.0", "tiktoken>=0.3.0"],
         "opentelemetry": ["opentelemetry-distro>=0.35b0"],
         "opentelemetry-experimental": [
-            "opentelemetry-distro~=0.40b0",
-            "opentelemetry-instrumentation-aiohttp-client~=0.40b0",
-            "opentelemetry-instrumentation-django~=0.40b0",
-            "opentelemetry-instrumentation-fastapi~=0.40b0",
-            "opentelemetry-instrumentation-flask~=0.40b0",
-            "opentelemetry-instrumentation-requests~=0.40b0",
-            "opentelemetry-instrumentation-sqlite3~=0.40b0",
-            "opentelemetry-instrumentation-urllib~=0.40b0",
+            # There's an umbrella package called
+            # opentelemetry-contrib-instrumentations that installs all
+            # available instrumentation packages, however it's broken in recent
+            # versions (after 0.41b0), see
+            # https://github.com/open-telemetry/opentelemetry-python-contrib/issues/2053
+            "opentelemetry-instrumentation-aio-pika==0.46b0",
+            "opentelemetry-instrumentation-aiohttp-client==0.46b0",
+            # "opentelemetry-instrumentation-aiohttp-server==0.46b0",  # broken package
+            "opentelemetry-instrumentation-aiopg==0.46b0",
+            "opentelemetry-instrumentation-asgi==0.46b0",
+            "opentelemetry-instrumentation-asyncio==0.46b0",
+            "opentelemetry-instrumentation-asyncpg==0.46b0",
+            "opentelemetry-instrumentation-aws-lambda==0.46b0",
+            "opentelemetry-instrumentation-boto==0.46b0",
+            "opentelemetry-instrumentation-boto3sqs==0.46b0",
+            "opentelemetry-instrumentation-botocore==0.46b0",
+            "opentelemetry-instrumentation-cassandra==0.46b0",
+            "opentelemetry-instrumentation-celery==0.46b0",
+            "opentelemetry-instrumentation-confluent-kafka==0.46b0",
+            "opentelemetry-instrumentation-dbapi==0.46b0",
+            "opentelemetry-instrumentation-django==0.46b0",
+            "opentelemetry-instrumentation-elasticsearch==0.46b0",
+            "opentelemetry-instrumentation-falcon==0.46b0",
+            "opentelemetry-instrumentation-fastapi==0.46b0",
+            "opentelemetry-instrumentation-flask==0.46b0",
+            "opentelemetry-instrumentation-grpc==0.46b0",
+            "opentelemetry-instrumentation-httpx==0.46b0",
+            "opentelemetry-instrumentation-jinja2==0.46b0",
+            "opentelemetry-instrumentation-kafka-python==0.46b0",
+            "opentelemetry-instrumentation-logging==0.46b0",
+            "opentelemetry-instrumentation-mysql==0.46b0",
+            "opentelemetry-instrumentation-mysqlclient==0.46b0",
+            "opentelemetry-instrumentation-pika==0.46b0",
+            "opentelemetry-instrumentation-psycopg==0.46b0",
+            "opentelemetry-instrumentation-psycopg2==0.46b0",
+            "opentelemetry-instrumentation-pymemcache==0.46b0",
+            "opentelemetry-instrumentation-pymongo==0.46b0",
+            "opentelemetry-instrumentation-pymysql==0.46b0",
+            "opentelemetry-instrumentation-pyramid==0.46b0",
+            "opentelemetry-instrumentation-redis==0.46b0",
+            "opentelemetry-instrumentation-remoulade==0.46b0",
+            "opentelemetry-instrumentation-requests==0.46b0",
+            "opentelemetry-instrumentation-sklearn==0.46b0",
+            "opentelemetry-instrumentation-sqlalchemy==0.46b0",
+            "opentelemetry-instrumentation-sqlite3==0.46b0",
+            "opentelemetry-instrumentation-starlette==0.46b0",
+            "opentelemetry-instrumentation-system-metrics==0.46b0",
+            "opentelemetry-instrumentation-threading==0.46b0",
+            "opentelemetry-instrumentation-tornado==0.46b0",
+            "opentelemetry-instrumentation-tortoiseorm==0.46b0",
+            "opentelemetry-instrumentation-urllib==0.46b0",
+            "opentelemetry-instrumentation-urllib3==0.46b0",
+            "opentelemetry-instrumentation-wsgi==0.46b0",
         ],
         "pure_eval": ["pure_eval", "executing", "asttokens"],
         "pymongo": ["pymongo>=3.1"],

tests/conftest.py

@@ -21,8 +21,11 @@
 
 import sentry_sdk
 from sentry_sdk.envelope import Envelope
-from sentry_sdk.integrations import _processed_integrations  # noqa: F401
-from sentry_sdk.profiler.transaction_profiler import teardown_profiler
+from sentry_sdk.integrations import (  # noqa: F401
+    _DEFAULT_INTEGRATIONS,
+    _processed_integrations,
+)
+from sentry_sdk.profiler import teardown_profiler
 from sentry_sdk.profiler.continuous_profiler import teardown_continuous_profiler
 from sentry_sdk.transport import Transport
 from sentry_sdk.utils import reraise
@@ -169,7 +172,13 @@ def reset_integrations():
     with a clean slate to ensure monkeypatching works well,
     but this also means some other stuff will be monkeypatched twice.
     """
-    global _processed_integrations
+    global _DEFAULT_INTEGRATIONS, _processed_integrations
+    try:
+        _DEFAULT_INTEGRATIONS.remove(
+            "sentry_sdk.integrations.opentelemetry.integration.OpenTelemetryIntegration"
+        )
+    except ValueError:
+        pass
     _processed_integrations.clear()