diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index 03657457e6..df05155391 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -164,6 +164,30 @@ class SPANDATA: Example: 16456 """ + CODE_FILEPATH = "code.filepath" + """ + The source code file name that identifies the code unit as uniquely as possible (preferably an absolute file path). + Example: "/app/myapplication/http/handler/server.py" + """ + + CODE_LINENO = "code.lineno" + """ + The line number in `code.filepath` best representing the operation. It SHOULD point within the code unit named in `code.function`. + Example: 42 + """ + + CODE_FUNCTION = "code.function" + """ + The method or function name, or equivalent (usually rightmost part of the code unit's name). + Example: "server_request" + """ + + CODE_NAMESPACE = "code.namespace" + """ + The "namespace" within which `code.function` is defined. Usually the qualified class or module name, such that `code.namespace` + some separator + `code.function` form a unique identifier for the code unit. + Example: "http.handler" + """ + class OP: CACHE_GET_ITEM = "cache.get_item" @@ -264,6 +288,8 @@ def __init__( max_value_length=DEFAULT_MAX_VALUE_LENGTH, # type: int enable_backpressure_handling=True, # type: bool error_sampler=None, # type: Optional[Callable[[Event, Hint], Union[float, bool]]] + enable_db_query_source=False, # type: bool + db_query_source_threshold_ms=100, # type: int spotlight=None, # type: Optional[Union[bool, str]] ): # type: (...) -> None diff --git a/sentry_sdk/tracing.py b/sentry_sdk/tracing.py index c32c0f6af4..26c413a34e 100644 --- a/sentry_sdk/tracing.py +++ b/sentry_sdk/tracing.py @@ -479,6 +479,8 @@ def finish(self, hub=None, end_timestamp=None): self.timestamp = datetime_utcnow() maybe_create_breadcrumbs_from_span(hub, self) + add_additional_span_data(hub, self) + return None def to_json(self): @@ -998,6 +1000,7 @@ async def my_async_function(): from sentry_sdk.tracing_utils import ( Baggage, EnvironHeaders, + add_additional_span_data, extract_sentrytrace_data, has_tracing_enabled, maybe_create_breadcrumbs_from_span, diff --git a/sentry_sdk/tracing_utils.py b/sentry_sdk/tracing_utils.py index 2a89145663..1beb48b538 100644 --- a/sentry_sdk/tracing_utils.py +++ b/sentry_sdk/tracing_utils.py @@ -1,14 +1,16 @@ -import re import contextlib +import re +import sys import sentry_sdk -from sentry_sdk.consts import OP +from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.utils import ( capture_internal_exceptions, Dsn, match_regex_list, to_string, is_sentry_url, + _is_external_source, ) from sentry_sdk._compat import PY2, iteritems from sentry_sdk._types import TYPE_CHECKING @@ -29,6 +31,8 @@ from typing import Optional from typing import Union + from types import FrameType + SENTRY_TRACE_REGEX = re.compile( "^[ \t]*" # whitespace @@ -162,6 +166,98 @@ def maybe_create_breadcrumbs_from_span(hub, span): ) +def add_query_source(hub, span): + # type: (sentry_sdk.Hub, sentry_sdk.tracing.Span) -> None + """ + Adds OTel compatible source code information to the span + """ + client = hub.client + if client is None: + return + + if span.timestamp is None or span.start_timestamp is None: + return + + should_add_query_source = client.options.get("enable_db_query_source", False) + if not should_add_query_source: + return + + duration = span.timestamp - span.start_timestamp + threshold = client.options.get("db_query_source_threshold_ms", 0) + slow_query = duration.microseconds > threshold * 1000 + + if not slow_query: + return + + project_root = client.options["project_root"] + + # Find the correct frame + frame = sys._getframe() # type: Union[FrameType, None] + while frame is not None: + try: + abs_path = frame.f_code.co_filename + except Exception: + abs_path = "" + + try: + namespace = frame.f_globals.get("__name__") + except Exception: + namespace = None + + is_sentry_sdk_frame = namespace is not None and namespace.startswith( + "sentry_sdk." + ) + if ( + abs_path.startswith(project_root) + and not _is_external_source(abs_path) + and not is_sentry_sdk_frame + ): + break + frame = frame.f_back + else: + frame = None + + # Set the data + if frame is not None: + try: + lineno = frame.f_lineno + except Exception: + lineno = None + if lineno is not None: + span.set_data(SPANDATA.CODE_LINENO, frame.f_lineno) + + try: + namespace = frame.f_globals.get("__name__") + except Exception: + namespace = None + if namespace is not None: + span.set_data(SPANDATA.CODE_NAMESPACE, namespace) + + try: + filepath = frame.f_code.co_filename + except Exception: + filepath = None + if filepath is not None: + span.set_data(SPANDATA.CODE_FILEPATH, frame.f_code.co_filename) + + try: + code_function = frame.f_code.co_name + except Exception: + code_function = None + + if code_function is not None: + span.set_data(SPANDATA.CODE_FUNCTION, frame.f_code.co_name) + + +def add_additional_span_data(hub, span): + # type: (sentry_sdk.Hub, sentry_sdk.tracing.Span) -> None + """ + Adds additional data to the span + """ + if span.op == OP.DB: + add_query_source(hub, span) + + def extract_sentrytrace_data(header): # type: (Optional[str]) -> Optional[Dict[str, Union[str, bool, None]]] """ diff --git a/tests/integrations/asyncpg/test_asyncpg.py b/tests/integrations/asyncpg/test_asyncpg.py index e9b2a9d740..c72144dd3a 100644 --- a/tests/integrations/asyncpg/test_asyncpg.py +++ b/tests/integrations/asyncpg/test_asyncpg.py @@ -27,8 +27,9 @@ from asyncpg import connect, Connection -from sentry_sdk import capture_message +from sentry_sdk import capture_message, start_transaction from sentry_sdk.integrations.asyncpg import AsyncPGIntegration +from sentry_sdk.consts import SPANDATA PG_CONNECTION_URI = "postgresql://{}:{}@{}/{}".format( @@ -460,3 +461,85 @@ async def test_connection_pool(sentry_init, capture_events) -> None: "type": "default", }, ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("enable_db_query_source", [None, False]) +async def test_query_source_disabled( + sentry_init, capture_events, enable_db_query_source +): + sentry_options = { + "integrations": [AsyncPGIntegration()], + "enable_tracing": True, + } + if enable_db_query_source is not None: + sentry_options["enable_db_query_source"] = enable_db_query_source + sentry_options["db_query_source_threshold_ms"] = 0 + + sentry_init(**sentry_options) + + events = capture_events() + + with start_transaction(name="test_transaction", sampled=True): + conn: Connection = await connect(PG_CONNECTION_URI) + + await conn.execute( + "INSERT INTO users(name, password, dob) VALUES ('Alice', 'secret', '1990-12-25')", + ) + + await conn.close() + + (event,) = events + + span = event["spans"][-1] + assert span["description"].startswith("INSERT INTO") + + data = span.get("data", {}) + + assert SPANDATA.CODE_LINENO not in data + assert SPANDATA.CODE_NAMESPACE not in data + assert SPANDATA.CODE_FILEPATH not in data + assert SPANDATA.CODE_FUNCTION not in data + + +@pytest.mark.asyncio +async def test_query_source(sentry_init, capture_events): + sentry_init( + integrations=[AsyncPGIntegration()], + enable_tracing=True, + enable_db_query_source=True, + db_query_source_threshold_ms=0, + ) + + events = capture_events() + + with start_transaction(name="test_transaction", sampled=True): + conn: Connection = await connect(PG_CONNECTION_URI) + + await conn.execute( + "INSERT INTO users(name, password, dob) VALUES ('Alice', 'secret', '1990-12-25')", + ) + + await conn.close() + + (event,) = events + + span = event["spans"][-1] + assert span["description"].startswith("INSERT INTO") + + data = span.get("data", {}) + + assert SPANDATA.CODE_LINENO in data + assert SPANDATA.CODE_NAMESPACE in data + assert SPANDATA.CODE_FILEPATH in data + assert SPANDATA.CODE_FUNCTION in data + + assert type(data.get(SPANDATA.CODE_LINENO)) == int + assert data.get(SPANDATA.CODE_LINENO) > 0 + assert ( + data.get(SPANDATA.CODE_NAMESPACE) == "tests.integrations.asyncpg.test_asyncpg" + ) + assert data.get(SPANDATA.CODE_FILEPATH).endswith( + "tests/integrations/asyncpg/test_asyncpg.py" + ) + assert data.get(SPANDATA.CODE_FUNCTION) == "test_query_source" diff --git a/tests/integrations/django/myapp/urls.py b/tests/integrations/django/myapp/urls.py index be5a40239e..0a62e4a076 100644 --- a/tests/integrations/django/myapp/urls.py +++ b/tests/integrations/django/myapp/urls.py @@ -57,6 +57,7 @@ def path(path, *args, **kwargs): path("template-test2", views.template_test2, name="template_test2"), path("template-test3", views.template_test3, name="template_test3"), path("postgres-select", views.postgres_select, name="postgres_select"), + path("postgres-select-slow", views.postgres_select_orm, name="postgres_select_orm"), path( "permission-denied-exc", views.permission_denied_exc, diff --git a/tests/integrations/django/myapp/views.py b/tests/integrations/django/myapp/views.py index 08262b4e8a..193147003b 100644 --- a/tests/integrations/django/myapp/views.py +++ b/tests/integrations/django/myapp/views.py @@ -193,6 +193,12 @@ def postgres_select(request, *args, **kwargs): return HttpResponse("ok") +@csrf_exempt +def postgres_select_orm(request, *args, **kwargs): + user = User.objects.using("postgres").all().first() + return HttpResponse("ok {}".format(user)) + + @csrf_exempt def permission_denied_exc(*args, **kwargs): raise PermissionDenied("bye") diff --git a/tests/integrations/django/test_db_query_data.py b/tests/integrations/django/test_db_query_data.py new file mode 100644 index 0000000000..1fa5ad4a8e --- /dev/null +++ b/tests/integrations/django/test_db_query_data.py @@ -0,0 +1,125 @@ +from __future__ import absolute_import + +import pytest + +try: + from django.urls import reverse +except ImportError: + from django.core.urlresolvers import reverse + +from django.db import connections + +from werkzeug.test import Client + +from sentry_sdk._compat import PY2 +from sentry_sdk.consts import SPANDATA +from sentry_sdk.integrations.django import DjangoIntegration + +from tests.conftest import unpack_werkzeug_response +from tests.integrations.django.utils import pytest_mark_django_db_decorator +from tests.integrations.django.myapp.wsgi import application + + +@pytest.fixture +def client(): + return Client(application) + + +@pytest.mark.forked +@pytest_mark_django_db_decorator(transaction=True) +@pytest.mark.parametrize("enable_db_query_source", [None, False]) +def test_query_source_disabled( + sentry_init, client, capture_events, enable_db_query_source +): + sentry_options = { + "integrations": [DjangoIntegration()], + "send_default_pii": True, + "traces_sample_rate": 1.0, + } + if enable_db_query_source is not None: + sentry_options["enable_db_query_source"] = enable_db_query_source + sentry_options["db_query_source_threshold_ms"] = 0 + + sentry_init(**sentry_options) + + if "postgres" not in connections: + pytest.skip("postgres tests disabled") + + # trigger Django to open a new connection by marking the existing one as None. + connections["postgres"].connection = None + + events = capture_events() + + _, status, _ = unpack_werkzeug_response(client.get(reverse("postgres_select_orm"))) + assert status == "200 OK" + + (event,) = events + for span in event["spans"]: + if span.get("op") == "db" and "auth_user" in span.get("description"): + data = span.get("data", {}) + + assert SPANDATA.CODE_LINENO not in data + assert SPANDATA.CODE_NAMESPACE not in data + assert SPANDATA.CODE_FILEPATH not in data + assert SPANDATA.CODE_FUNCTION not in data + break + else: + raise AssertionError("No db span found") + + +@pytest.mark.forked +@pytest_mark_django_db_decorator(transaction=True) +def test_query_source(sentry_init, client, capture_events): + sentry_init( + integrations=[DjangoIntegration()], + send_default_pii=True, + traces_sample_rate=1.0, + enable_db_query_source=True, + db_query_source_threshold_ms=0, + ) + + if "postgres" not in connections: + pytest.skip("postgres tests disabled") + + # trigger Django to open a new connection by marking the existing one as None. + connections["postgres"].connection = None + + events = capture_events() + + _, status, _ = unpack_werkzeug_response(client.get(reverse("postgres_select_orm"))) + assert status == "200 OK" + + (event,) = events + for span in event["spans"]: + if span.get("op") == "db" and "auth_user" in span.get("description"): + data = span.get("data", {}) + + assert SPANDATA.CODE_LINENO in data + assert SPANDATA.CODE_NAMESPACE in data + assert SPANDATA.CODE_FILEPATH in data + assert SPANDATA.CODE_FUNCTION in data + + assert type(data.get(SPANDATA.CODE_LINENO)) == int + assert data.get(SPANDATA.CODE_LINENO) > 0 + + if PY2: + assert ( + data.get(SPANDATA.CODE_NAMESPACE) + == "tests.integrations.django.test_db_query_data" + ) + assert data.get(SPANDATA.CODE_FILEPATH).endswith( + "tests/integrations/django/test_db_query_data.py" + ) + assert data.get(SPANDATA.CODE_FUNCTION) == "test_query_source" + else: + assert ( + data.get(SPANDATA.CODE_NAMESPACE) + == "tests.integrations.django.myapp.views" + ) + assert data.get(SPANDATA.CODE_FILEPATH).endswith( + "tests/integrations/django/myapp/views.py" + ) + assert data.get(SPANDATA.CODE_FUNCTION) == "postgres_select_orm" + break + else: + raise AssertionError("No db span found") diff --git a/tests/integrations/sqlalchemy/test_sqlalchemy.py b/tests/integrations/sqlalchemy/test_sqlalchemy.py index eb1792b3be..cfcf139616 100644 --- a/tests/integrations/sqlalchemy/test_sqlalchemy.py +++ b/tests/integrations/sqlalchemy/test_sqlalchemy.py @@ -225,3 +225,109 @@ def test_engine_name_not_string(sentry_init): with engine.connect() as con: con.execute(text("SELECT 0")) + + +@pytest.mark.parametrize("enable_db_query_source", [None, False]) +def test_query_source_disabled(sentry_init, capture_events, enable_db_query_source): + sentry_options = { + "integrations": [SqlalchemyIntegration()], + "enable_tracing": True, + } + if enable_db_query_source is not None: + sentry_options["enable_db_query_source"] = enable_db_query_source + sentry_options["db_query_source_threshold_ms"] = 0 + + sentry_init(**sentry_options) + + events = capture_events() + + with start_transaction(name="test_transaction", sampled=True): + Base = declarative_base() # noqa: N806 + + class Person(Base): + __tablename__ = "person" + id = Column(Integer, primary_key=True) + name = Column(String(250), nullable=False) + + engine = create_engine("sqlite:///:memory:") + Base.metadata.create_all(engine) + + Session = sessionmaker(bind=engine) # noqa: N806 + session = Session() + + bob = Person(name="Bob") + session.add(bob) + + assert session.query(Person).first() == bob + + (event,) = events + + for span in event["spans"]: + if span.get("op") == "db" and span.get("description").startswith( + "SELECT person" + ): + data = span.get("data", {}) + + assert SPANDATA.CODE_LINENO not in data + assert SPANDATA.CODE_NAMESPACE not in data + assert SPANDATA.CODE_FILEPATH not in data + assert SPANDATA.CODE_FUNCTION not in data + break + else: + raise AssertionError("No db span found") + + +def test_query_source(sentry_init, capture_events): + sentry_init( + integrations=[SqlalchemyIntegration()], + enable_tracing=True, + enable_db_query_source=True, + db_query_source_threshold_ms=0, + ) + events = capture_events() + + with start_transaction(name="test_transaction", sampled=True): + Base = declarative_base() # noqa: N806 + + class Person(Base): + __tablename__ = "person" + id = Column(Integer, primary_key=True) + name = Column(String(250), nullable=False) + + engine = create_engine("sqlite:///:memory:") + Base.metadata.create_all(engine) + + Session = sessionmaker(bind=engine) # noqa: N806 + session = Session() + + bob = Person(name="Bob") + session.add(bob) + + assert session.query(Person).first() == bob + + (event,) = events + + for span in event["spans"]: + if span.get("op") == "db" and span.get("description").startswith( + "SELECT person" + ): + data = span.get("data", {}) + + assert SPANDATA.CODE_LINENO in data + assert SPANDATA.CODE_NAMESPACE in data + assert SPANDATA.CODE_FILEPATH in data + assert SPANDATA.CODE_FUNCTION in data + + assert type(data.get(SPANDATA.CODE_LINENO)) == int + assert data.get(SPANDATA.CODE_LINENO) > 0 + assert ( + data.get(SPANDATA.CODE_NAMESPACE) + == "tests.integrations.sqlalchemy.test_sqlalchemy" + ) + assert data.get(SPANDATA.CODE_FILEPATH).endswith( + "tests/integrations/sqlalchemy/test_sqlalchemy.py" + ) + assert data.get(SPANDATA.CODE_FUNCTION) == "test_query_source" + break + else: + raise AssertionError("No db span found")