diff --git a/sentry_sdk/__init__.py b/sentry_sdk/__init__.py old mode 100755 new mode 100644 diff --git a/sentry_sdk/_compat.py b/sentry_sdk/_compat.py old mode 100755 new mode 100644 index c94ef6d..e357c96 --- a/sentry_sdk/_compat.py +++ b/sentry_sdk/_compat.py @@ -8,6 +8,10 @@ from typing import Any from typing import Type + from typing import TypeVar + + T = TypeVar("T") + PY2 = sys.version_info[0] == 2 @@ -23,6 +27,7 @@ iteritems = lambda x: x.iteritems() # noqa: B301 def implements_str(cls): + # type: (T) -> T cls.__unicode__ = cls.__str__ cls.__str__ = lambda x: unicode(x).encode("utf-8") # noqa return cls @@ -40,10 +45,8 @@ def implements_str(cls): int_types = (int,) # noqa iteritems = lambda x: x.items() - def _identity(x): - return x - def implements_str(x): + # type: (T) -> T return x def reraise(tp, value, tb=None): @@ -55,8 +58,10 @@ def reraise(tp, value, tb=None): def with_metaclass(meta, *bases): + # type: (Any, *Any) -> Any class metaclass(type): - def __new__(cls, name, this_bases, d): + def __new__(metacls, name, this_bases, d): + # type: (Any, Any, Any, Any) -> Any return meta(name, bases, d) return type.__new__(metaclass, "temporary_class", (), {}) diff --git a/sentry_sdk/_types.py b/sentry_sdk/_types.py old mode 100755 new mode 100644 index 99654e9..6f9af8d --- a/sentry_sdk/_types.py +++ b/sentry_sdk/_types.py @@ -26,3 +26,6 @@ EventProcessor = Callable[[Event, Hint], Optional[Event]] ErrorProcessor = Callable[[Event, ExcInfo], Optional[Event]] BreadcrumbProcessor = Callable[[Breadcrumb, BreadcrumbHint], Optional[Breadcrumb]] + + # https://github.com/python/mypy/issues/5710 + NotImplementedType = Any diff --git a/sentry_sdk/api.py b/sentry_sdk/api.py old mode 100755 new mode 100644 index 93d8137..6ecb33b --- a/sentry_sdk/api.py +++ b/sentry_sdk/api.py @@ -8,6 +8,7 @@ if MYPY: from typing import Any + from typing import Dict from typing import Optional from typing import overload from typing import Callable @@ -15,6 +16,7 @@ from typing import ContextManager from sentry_sdk._types import Event, Hint, Breadcrumb, BreadcrumbHint + from sentry_sdk.tracing import Span T = TypeVar("T") F = TypeVar("F", bound=Callable[..., Any]) @@ -34,6 +36,12 @@ def overload(x): "push_scope", "flush", "last_event_id", + "start_span", + "set_tag", + "set_context", + "set_extra", + "set_user", + "set_level", ] @@ -46,6 +54,15 @@ def hubmethod(f): return f +def scopemethod(f): + # type: (F) -> F + f.__doc__ = "%s\n\n%s" % ( + "Alias for :py:meth:`sentry_sdk.Scope.%s`" % f.__name__, + inspect.getdoc(getattr(Scope, f.__name__)), + ) + return f + + @hubmethod def capture_event( event, # type: Event @@ -161,6 +178,46 @@ def inner(): return None +@scopemethod # noqa +def set_tag(key, value): + # type: (str, Any) -> None + hub = Hub.current + if hub is not None: + hub.scope.set_tag(key, value) + + +@scopemethod # noqa +def set_context(key, value): + # type: (str, Any) -> None + hub = Hub.current + if hub is not None: + hub.scope.set_context(key, value) + + +@scopemethod # noqa +def set_extra(key, value): + # type: (str, Any) -> None + hub = Hub.current + if hub is not None: + hub.scope.set_extra(key, value) + + +@scopemethod # noqa +def set_user(value): + # type: (Dict[str, Any]) -> None + hub = Hub.current + if hub is not None: + hub.scope.set_user(value) + + +@scopemethod # noqa +def set_level(value): + # type: (str) -> None + hub = Hub.current + if hub is not None: + hub.scope.set_level(value) + + @hubmethod def flush( timeout=None, # type: Optional[float] @@ -179,3 +236,15 @@ def last_event_id(): if hub is not None: return hub.last_event_id() return None + + +@hubmethod +def start_span( + span=None, # type: Optional[Span] + **kwargs # type: Any +): + # type: (...) -> Span + + # TODO: All other functions in this module check for + # `Hub.current is None`. That actually should never happen? + return Hub.current.start_span(span=span, **kwargs) diff --git a/sentry_sdk/client.py b/sentry_sdk/client.py old mode 100755 new mode 100644 index 1c2a379..e83c8a0 --- a/sentry_sdk/client.py +++ b/sentry_sdk/client.py @@ -10,9 +10,10 @@ get_type_name, capture_internal_exceptions, current_stacktrace, + disable_capture_event, logger, ) -from sentry_sdk.serializer import Serializer +from sentry_sdk.serializer import serialize from sentry_sdk.transport import make_transport from sentry_sdk.consts import DEFAULT_OPTIONS, SDK_INFO, ClientConstructor from sentry_sdk.integrations import setup_integrations @@ -42,9 +43,9 @@ def _get_options(*args, **kwargs): dsn = None rv = dict(DEFAULT_OPTIONS) - options = dict(*args, **kwargs) # type: ignore + options = dict(*args, **kwargs) if dsn is not None and options.get("dsn") is None: - options["dsn"] = dsn # type: ignore + options["dsn"] = dsn for key, value in iteritems(options): if key not in rv: @@ -63,7 +64,7 @@ def _get_options(*args, **kwargs): if rv["server_name"] is None and hasattr(socket, "gethostname"): rv["server_name"] = socket.gethostname() - return rv # type: ignore + return rv class _Client(object): @@ -74,15 +75,28 @@ class _Client(object): """ def __init__(self, *args, **kwargs): - # type: (*Optional[str], **Any) -> None + # type: (*Any, **Any) -> None + self.options = get_options(*args, **kwargs) # type: Dict[str, Any] + self._init_impl() + + def __getstate__(self): + # type: () -> Any + return {"options": self.options} + + def __setstate__(self, state): + # type: (Any) -> None + self.options = state["options"] + self._init_impl() + + def _init_impl(self): + # type: () -> None old_debug = _client_init_debug.get(False) try: - self.options = options = get_options(*args, **kwargs) # type: ignore - _client_init_debug.set(options["debug"]) - self.transport = make_transport(options) + _client_init_debug.set(self.options["debug"]) + self.transport = make_transport(self.options) request_bodies = ("always", "never", "small", "medium") - if options["request_bodies"] not in request_bodies: + if self.options["request_bodies"] not in request_bodies: raise ValueError( "Invalid value for request_bodies. Must be one of {}".format( request_bodies @@ -90,7 +104,8 @@ def __init__(self, *args, **kwargs): ) self.integrations = setup_integrations( - options["integrations"], with_defaults=options["default_integrations"] + self.options["integrations"], + with_defaults=self.options["default_integrations"], ) finally: _client_init_debug.set(old_debug) @@ -108,6 +123,7 @@ def _prepare_event( scope, # type: Optional[Scope] ): # type: (...) -> Optional[Event] + if event.get("timestamp") is None: event["timestamp"] = datetime.utcnow() @@ -139,8 +155,8 @@ def _prepare_event( } for key in "release", "environment", "server_name", "dist": - if event.get(key) is None and self.options[key] is not None: # type: ignore - event[key] = text_type(self.options[key]).strip() # type: ignore + if event.get(key) is None and self.options[key] is not None: + event[key] = text_type(self.options[key]).strip() if event.get("sdk") is None: sdk_info = dict(SDK_INFO) sdk_info["integrations"] = sorted(self.integrations.keys()) @@ -156,7 +172,7 @@ def _prepare_event( # Postprocess the event here so that annotated types do # generally not surface in before_send if event is not None: - event = Serializer().serialize_event(event) + event = serialize(event) before_send = self.options["before_send"] if before_send is not None: @@ -185,7 +201,7 @@ def _is_ignored_error(self, event, hint): if errcls == full_name or errcls == type_name: return True else: - if issubclass(exc_info[0], errcls): # type: ignore + if issubclass(exc_info[0], errcls): return True return False @@ -226,20 +242,23 @@ def capture_event( :returns: An event ID. May be `None` if there is no DSN set or of if the SDK decided to discard the event for other reasons. In such situations setting `debug=True` on `init()` may help. """ + if disable_capture_event.get(False): + return None + if self.transport is None: return None if hint is None: hint = {} - rv = event.get("event_id") - if rv is None: - event["event_id"] = rv = uuid.uuid4().hex + event_id = event.get("event_id") + if event_id is None: + event["event_id"] = event_id = uuid.uuid4().hex if not self._should_capture(event, hint, scope): return None - event = self._prepare_event(event, hint, scope) - if event is None: + event_opt = self._prepare_event(event, hint, scope) + if event_opt is None: return None - self.transport.capture_event(event) - return rv + self.transport.capture_event(event_opt) + return event_id def close( self, diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py old mode 100755 new mode 100644 index ae0a650..9c8f82f --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -8,6 +8,7 @@ from typing import Type from typing import Dict from typing import Any + from typing import Sequence from sentry_sdk.transport import Transport from sentry_sdk.integrations import Integration @@ -27,9 +28,9 @@ def __init__( environment=None, # type: Optional[str] server_name=None, # type: Optional[str] shutdown_timeout=2, # type: int - integrations=[], # type: List[Integration] - in_app_include=[], # type: List[str] - in_app_exclude=[], # type: List[str] + integrations=[], # type: Sequence[Integration] # noqa: B006 + in_app_include=[], # type: List[str] # noqa: B006 + in_app_exclude=[], # type: List[str] # noqa: B006 default_integrations=True, # type: bool dist=None, # type: Optional[str] transport=None, # type: Optional[Union[Transport, Type[Transport], Callable[[Event], None]]] @@ -37,7 +38,7 @@ def __init__( send_default_pii=False, # type: bool http_proxy=None, # type: Optional[str] https_proxy=None, # type: Optional[str] - ignore_errors=[], # type: List[Union[type, str]] + ignore_errors=[], # type: List[Union[type, str]] # noqa: B006 request_bodies="medium", # type: str before_send=None, # type: Optional[EventProcessor] before_breadcrumb=None, # type: Optional[BreadcrumbProcessor] @@ -48,6 +49,7 @@ def __init__( # DO NOT ENABLE THIS RIGHT NOW UNLESS YOU WANT TO EXCEED YOUR EVENT QUOTA IMMEDIATELY traces_sample_rate=0.0, # type: float traceparent_v2=False, # type: bool + _experiments={}, # type: Dict[str, Any] # noqa: B006 ): # type: (...) -> None pass @@ -58,7 +60,7 @@ def _get_default_options(): import inspect if hasattr(inspect, "getfullargspec"): - getargspec = inspect.getfullargspec # type: ignore + getargspec = inspect.getfullargspec else: getargspec = inspect.getargspec # type: ignore @@ -70,7 +72,7 @@ def _get_default_options(): del _get_default_options -VERSION = "0.10.0" +VERSION = "0.13.5" SDK_INFO = { "name": "sentry.python", "version": VERSION, diff --git a/sentry_sdk/debug.py b/sentry_sdk/debug.py old mode 100755 new mode 100644 diff --git a/sentry_sdk/hub.py b/sentry_sdk/hub.py old mode 100755 new mode 100644 index f022966..0849d46 --- a/sentry_sdk/hub.py +++ b/sentry_sdk/hub.py @@ -1,16 +1,14 @@ import copy import random import sys -import weakref from datetime import datetime from contextlib import contextmanager -from warnings import warn from sentry_sdk._compat import with_metaclass from sentry_sdk.scope import Scope from sentry_sdk.client import Client -from sentry_sdk.tracing import Span, maybe_create_breadcrumbs_from_span +from sentry_sdk.tracing import Span from sentry_sdk.utils import ( exc_info_from_error, event_from_exception, @@ -46,8 +44,7 @@ def overload(x): return x -_local = ContextVar("sentry_current_hub") # type: ignore -_initial_client = None # type: Optional[weakref.ReferenceType[Client]] +_local = ContextVar("sentry_current_hub") def _should_send_default_pii(): @@ -80,12 +77,9 @@ def _init(*args, **kwargs): This takes the same arguments as the client constructor. """ - global _initial_client client = Client(*args, **kwargs) # type: ignore Hub.current.bind_client(client) rv = _InitGuard(client) - if client is not None: - _initial_client = weakref.ref(client) return rv @@ -112,7 +106,7 @@ class init(ClientConstructor, ContextManager[Any]): class HubMeta(type): @property - def current(self): + def current(cls): # type: () -> Hub """Returns the current instance of the hub.""" rv = _local.get(None) @@ -122,23 +116,12 @@ def current(self): return rv @property - def main(self): + def main(cls): # type: () -> Hub """Returns the main instance of the hub.""" return GLOBAL_HUB -class _HubManager(object): - def __init__(self, hub): - # type: (Hub) -> None - self._old = Hub.current - _local.set(hub) - - def __exit__(self, exc_type, exc_value, tb): - # type: (Any, Any, Any) -> None - _local.set(self._old) - - class _ScopeManager(object): def __init__(self, hub): # type: (Hub) -> None @@ -273,32 +256,18 @@ def get_integration( if rv is not None: return rv - if _initial_client is not None: - initial_client = _initial_client() - else: - initial_client = None - - if ( - initial_client is not None - and initial_client is not client - and initial_client.integrations.get(integration_name) is not None - ): - warning = ( - "Integration %r attempted to run but it was only " - "enabled on init() but not the client that " - "was bound to the current flow. Earlier versions of " - "the SDK would consider these integrations enabled but " - "this is no longer the case." % (name_or_class,) - ) - warn(Warning(warning), stacklevel=3) - logger.warning(warning) - @property def client(self): # type: () -> Optional[Client] """Returns the current client on the hub.""" return self._stack[-1][0] + @property + def scope(self): + # type: () -> Scope + """Returns the current scope on the hub.""" + return self._stack[-1][1] + def last_event_id(self): # type: () -> Optional[str] """Returns the last event ID.""" @@ -358,10 +327,10 @@ def capture_exception( client = self.client if client is None: return None - if error is None: - exc_info = sys.exc_info() - else: + if error is not None: exc_info = exc_info_from_error(error) + else: + exc_info = sys.exc_info() event, hint = event_from_exception(exc_info, client_options=client.options) try: @@ -381,7 +350,7 @@ def _capture_internal_exception( These exceptions do not end up in Sentry and are just logged instead. """ - logger.error("Internal error in sentry_sdk", exc_info=exc_info) # type: ignore + logger.error("Internal error in sentry_sdk", exc_info=exc_info) def add_breadcrumb( self, @@ -429,49 +398,31 @@ def add_breadcrumb( while len(scope._breadcrumbs) > max_breadcrumbs: scope._breadcrumbs.popleft() - @contextmanager - def span( - self, - span=None, # type: Optional[Span] - **kwargs # type: Any - ): - # type: (...) -> Generator[Span, None, None] - # TODO: Document - span = self.start_span(span=span, **kwargs) - - _, scope = self._stack[-1] - old_span = scope.span - scope.span = span - - try: - yield span - except Exception: - span.set_tag("error", True) - raise - else: - span.set_tag("error", False) - finally: - try: - span.finish() - maybe_create_breadcrumbs_from_span(self, span) - self.finish_span(span) - except Exception: - self._capture_internal_exception(sys.exc_info()) - scope.span = old_span - def start_span( self, span=None, # type: Optional[Span] **kwargs # type: Any ): # type: (...) -> Span - # TODO: Document + """ + Create a new span whose parent span is the currently active + span, if any. The return value is the span object that can + be used as a context manager to start and stop timing. + + Note that you will not see any span that is not contained + within a transaction. Create a transaction with + ``start_span(transaction="my transaction")`` if an + integration doesn't already do this for you. + """ client, scope = self._stack[-1] + kwargs.setdefault("hub", self) + if span is None: - if scope.span is not None: - span = scope.span.new_span(**kwargs) + span = scope.span + if span is not None: + span = span.new_span(**kwargs) else: span = Span(**kwargs) @@ -479,47 +430,13 @@ def start_span( sample_rate = client and client.options["traces_sample_rate"] or 0 span.sampled = random.random() < sample_rate - return span - - def finish_span( - self, span # type: Span - ): - # type: (...) -> Optional[str] - # TODO: Document - if span.timestamp is None: - # This transaction is not yet finished so we just finish it. - span.finish() - - if span.transaction is None: - # If this has no transaction set we assume there's a parent - # transaction for this span that would be flushed out eventually. - return None - - if self.client is None: - # We have no client and therefore nowhere to send this transaction - # event. - return None - - if not span.sampled: - # At this point a `sampled = None` should have already been - # resolved to a concrete decision. If `sampled` is `None`, it's - # likely that somebody used `with Hub.span(..)` on a - # non-transaction span and later decided to make it a transaction. - assert ( - span.sampled is not None - ), "Need to set transaction when entering span!" - return None + if span.sampled: + max_spans = ( + client and client.options["_experiments"].get("max_spans") or 1000 + ) + span.init_finished_spans(maxlen=max_spans) - return self.capture_event( - { - "type": "transaction", - "transaction": span.transaction, - "contexts": {"trace": span.get_trace_context()}, - "timestamp": span.timestamp, - "start_timestamp": span.start_timestamp, - "spans": [s.to_json() for s in span._finished_spans if s is not span], - } - ) + return span @overload # noqa def push_scope( @@ -560,8 +477,6 @@ def push_scope( # noqa return _ScopeManager(self) - scope = push_scope - def pop_scope_unsafe(self): # type: () -> Tuple[Optional[Client], Scope] """ @@ -634,7 +549,9 @@ def iter_trace_propagation_headers(self): # type: () -> Generator[Tuple[str, str], None, None] # TODO: Document client, scope = self._stack[-1] - if scope._span is None: + span = scope.span + + if span is None: return propagate_traces = client and client.options["propagate_traces"] @@ -642,9 +559,9 @@ def iter_trace_propagation_headers(self): return if client and client.options["traceparent_v2"]: - traceparent = scope._span.to_traceparent() + traceparent = span.to_traceparent() else: - traceparent = scope._span.to_legacy_traceparent() + traceparent = span.to_legacy_traceparent() yield "sentry-trace", traceparent diff --git a/sentry_sdk/integrations/__init__.py b/sentry_sdk/integrations/__init__.py old mode 100755 new mode 100644 index 92229a3..18c8069 --- a/sentry_sdk/integrations/__init__.py +++ b/sentry_sdk/integrations/__init__.py @@ -72,7 +72,7 @@ def setup_integrations(integrations, with_defaults=True): instance = integration_cls() integrations[instance.identifier] = instance - for identifier, integration in iteritems(integrations): # type: ignore + for identifier, integration in iteritems(integrations): with _installer_lock: if identifier not in _installed_integrations: logger.debug( @@ -82,7 +82,7 @@ def setup_integrations(integrations, with_defaults=True): type(integration).setup_once() except NotImplementedError: if getattr(integration, "install", None) is not None: - logger.warn( + logger.warning( "Integration %s: The install method is " "deprecated. Use `setup_once`.", identifier, diff --git a/sentry_sdk/integrations/_sql_common.py b/sentry_sdk/integrations/_sql_common.py deleted file mode 100755 index 7096c23..0000000 --- a/sentry_sdk/integrations/_sql_common.py +++ /dev/null @@ -1,81 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import absolute_import - -from sentry_sdk.utils import format_and_strip, safe_repr - -if False: - from typing import Any - from typing import Dict - from typing import List - from typing import Tuple - from typing import Optional - - -class _FormatConverter(object): - def __init__(self, param_mapping): - # type: (Dict[str, int]) -> None - - self.param_mapping = param_mapping - self.params = [] # type: List[Any] - - def __getitem__(self, val): - # type: (str) -> str - self.params.append(self.param_mapping.get(val)) - return "%s" - - -def _format_sql_impl(sql, params): - # type: (Any, Any) -> Tuple[str, List[str]] - rv = [] - - if isinstance(params, dict): - # convert sql with named parameters to sql with unnamed parameters - conv = _FormatConverter(params) - if params: - sql = sql % conv - params = conv.params - else: - params = () - - for param in params or (): - if param is None: - rv.append("NULL") - param = safe_repr(param) - rv.append(param) - - return sql, rv - - -def format_sql(sql, params, cursor): - # type: (str, List[Any], Any) -> Optional[str] - - real_sql = None - real_params = None - - try: - # Prefer our own SQL formatting logic because it's the only one that - # has proper value trimming. - real_sql, real_params = _format_sql_impl(sql, params) - if real_sql: - real_sql = format_and_strip(real_sql, real_params) - except Exception: - pass - - if not real_sql and hasattr(cursor, "mogrify"): - # If formatting failed and we're using psycopg2, it could be that we're - # looking at a query that uses Composed objects. Use psycopg2's mogrify - # function to format the query. We lose per-parameter trimming but gain - # accuracy in formatting. - # - # This is intentionally the second choice because we assume Composed - # queries are not widely used, while per-parameter trimming is - # generally highly desirable. - try: - if hasattr(cursor, "mogrify"): - real_sql = cursor.mogrify(sql, params) - if isinstance(real_sql, bytes): - real_sql = real_sql.decode(cursor.connection.encoding) - except Exception: - pass - - return real_sql or None diff --git a/sentry_sdk/integrations/_wsgi_common.py b/sentry_sdk/integrations/_wsgi_common.py old mode 100755 new mode 100644 index cb626a5..f874663 --- a/sentry_sdk/integrations/_wsgi_common.py +++ b/sentry_sdk/integrations/_wsgi_common.py @@ -7,6 +7,8 @@ from sentry_sdk._types import MYPY if MYPY: + import sentry_sdk + from typing import Any from typing import Dict from typing import Optional @@ -28,6 +30,19 @@ ) +def request_body_within_bounds(client, content_length): + # type: (Optional[sentry_sdk.Client], int) -> bool + if client is None: + return False + + bodies = client.options["request_bodies"] + return not ( + bodies == "never" + or (bodies == "small" and content_length > 10 ** 3) + or (bodies == "medium" and content_length > 10 ** 4) + ) + + class RequestExtractor(object): def __init__(self, request): # type: (Any) -> None @@ -42,17 +57,12 @@ def extract_into_event(self, event): data = None # type: Optional[Union[AnnotatedValue, Dict[str, Any]]] content_length = self.content_length() - request_info = event.setdefault("request", {}) + request_info = event.get("request", {}) if _should_send_default_pii(): request_info["cookies"] = dict(self.cookies()) - bodies = client.options["request_bodies"] - if ( - bodies == "never" - or (bodies == "small" and content_length > 10 ** 3) - or (bodies == "medium" and content_length > 10 ** 4) - ): + if not request_body_within_bounds(client, content_length): data = AnnotatedValue( "", {"rem": [["!config", "x", 0, content_length]], "len": content_length}, @@ -67,9 +77,12 @@ def extract_into_event(self, event): {"rem": [["!raw", "x", 0, content_length]], "len": content_length}, ) else: - return + data = None + + if data is not None: + request_info["data"] = data - request_info["data"] = data + event["request"] = request_info def content_length(self): # type: () -> int @@ -79,12 +92,15 @@ def content_length(self): return 0 def cookies(self): + # type: () -> Dict[str, Any] raise NotImplementedError() def raw_data(self): + # type: () -> Optional[Union[str, bytes]] raise NotImplementedError() def form(self): + # type: () -> Optional[Dict[str, Any]] raise NotImplementedError() def parsed_body(self): @@ -110,28 +126,37 @@ def is_json(self): def json(self): # type: () -> Optional[Any] try: - if self.is_json(): - raw_data = self.raw_data() - if not isinstance(raw_data, text_type): - raw_data = raw_data.decode("utf-8") + if not self.is_json(): + return None + + raw_data = self.raw_data() + if raw_data is None: + return None + + if isinstance(raw_data, text_type): return json.loads(raw_data) + else: + return json.loads(raw_data.decode("utf-8")) except ValueError: pass return None def files(self): + # type: () -> Optional[Dict[str, Any]] raise NotImplementedError() def size_of_file(self, file): + # type: (Any) -> int raise NotImplementedError() def env(self): + # type: () -> Dict[str, Any] raise NotImplementedError() def _is_json_content_type(ct): - # type: (str) -> bool + # type: (Optional[str]) -> bool mt = (ct or "").split(";", 1)[0] return ( mt == "application/json" diff --git a/sentry_sdk/integrations/aiohttp.py b/sentry_sdk/integrations/aiohttp.py old mode 100755 new mode 100644 index 5d09530..20b1a71 --- a/sentry_sdk/integrations/aiohttp.py +++ b/sentry_sdk/integrations/aiohttp.py @@ -5,26 +5,36 @@ from sentry_sdk.hub import Hub from sentry_sdk.integrations import Integration from sentry_sdk.integrations.logging import ignore_logger -from sentry_sdk.integrations._wsgi_common import _filter_headers +from sentry_sdk.integrations._wsgi_common import ( + _filter_headers, + request_body_within_bounds, +) +from sentry_sdk.tracing import Span from sentry_sdk.utils import ( capture_internal_exceptions, event_from_exception, + transaction_from_function, HAS_REAL_CONTEXTVARS, + AnnotatedValue, ) import asyncio -from aiohttp.web import Application, HTTPException # type: ignore +from aiohttp.web import Application, HTTPException, UrlDispatcher from sentry_sdk._types import MYPY if MYPY: - from aiohttp.web_request import Request # type: ignore + from aiohttp.web_request import Request + from aiohttp.abc import AbstractMatchInfo from typing import Any from typing import Dict + from typing import Optional from typing import Tuple from typing import Callable + from typing import Union from sentry_sdk.utils import ExcInfo + from sentry_sdk._types import EventProcessor class AioHttpIntegration(Integration): @@ -60,14 +70,28 @@ async def inner(): scope.clear_breadcrumbs() scope.add_event_processor(_make_request_processor(weak_request)) - try: - response = await old_handle(self, request) - except HTTPException: - raise - except Exception: - reraise(*_capture_exception(hub)) - - return response + span = Span.continue_from_headers(request.headers) + span.op = "http.server" + # If this transaction name makes it to the UI, AIOHTTP's + # URL resolver did not find a route or died trying. + span.transaction = "generic AIOHTTP request" + + with hub.start_span(span): + try: + response = await old_handle(self, request) + except HTTPException as e: + span.set_http_status(e.status_code) + raise + except asyncio.CancelledError: + span.set_status("cancelled") + raise + except Exception: + # This will probably map to a 500 but seems like we + # have no way to tell. Do not set span status. + reraise(*_capture_exception(hub)) + + span.set_http_status(response.status) + return response # Explicitly wrap in task such that current contextvar context is # copied. Just doing `return await inner()` will leak scope data @@ -76,9 +100,30 @@ async def inner(): Application._handle = sentry_app_handle + old_urldispatcher_resolve = UrlDispatcher.resolve + + async def sentry_urldispatcher_resolve(self, request): + # type: (UrlDispatcher, Request) -> AbstractMatchInfo + rv = await old_urldispatcher_resolve(self, request) + + name = None + + try: + name = transaction_from_function(rv.handler) + except Exception: + pass + + if name is not None: + with Hub.current.configure_scope() as scope: + scope.transaction = name + + return rv + + UrlDispatcher.resolve = sentry_urldispatcher_resolve + def _make_request_processor(weak_request): - # type: (Callable[[], Request]) -> Callable + # type: (Callable[[], Request]) -> EventProcessor def aiohttp_processor( event, # type: Dict[str, Any] hint, # type: Dict[str, Tuple[type, BaseException, Any]] @@ -89,9 +134,6 @@ def aiohttp_processor( return event with capture_internal_exceptions(): - # TODO: Figure out what to do with request body. Methods on request - # are async, but event processors are not. - request_info = event.setdefault("request", {}) request_info["url"] = "%s://%s%s" % ( @@ -103,8 +145,15 @@ def aiohttp_processor( request_info["query_string"] = request.query_string request_info["method"] = request.method request_info["env"] = {"REMOTE_ADDR": request.remote} + + hub = Hub.current request_info["headers"] = _filter_headers(dict(request.headers)) + # Just attach raw data here if it is within bounds, if available. + # Unfortunately there's no way to get structured data from aiohttp + # without awaiting on some coroutine. + request_info["data"] = get_aiohttp_request_data(hub, request) + return event return aiohttp_processor @@ -120,3 +169,29 @@ def _capture_exception(hub): ) hub.capture_event(event, hint=hint) return exc_info + + +BODY_NOT_READ_MESSAGE = "[Can't show request body due to implementation details.]" + + +def get_aiohttp_request_data(hub, request): + # type: (Hub, Request) -> Union[Optional[str], AnnotatedValue] + bytes_body = request._read_bytes + + if bytes_body is not None: + # we have body to show + if not request_body_within_bounds(hub.client, len(bytes_body)): + + return AnnotatedValue( + "", + {"rem": [["!config", "x", 0, len(bytes_body)]], "len": len(bytes_body)}, + ) + encoding = request.charset or "utf-8" + return bytes_body.decode(encoding, "replace") + + if request.can_read_body: + # body exists but we can't show it + return BODY_NOT_READ_MESSAGE + + # request has no body + return None diff --git a/sentry_sdk/integrations/argv.py b/sentry_sdk/integrations/argv.py old mode 100755 new mode 100644 diff --git a/sentry_sdk/integrations/asgi.py b/sentry_sdk/integrations/asgi.py new file mode 100644 index 0000000..762634f --- /dev/null +++ b/sentry_sdk/integrations/asgi.py @@ -0,0 +1,194 @@ +""" +An ASGI middleware. + +Based on Tom Christie's `sentry-asgi `_. +""" + +import asyncio +import functools +import inspect +import urllib + +from sentry_sdk._types import MYPY +from sentry_sdk.hub import Hub, _should_send_default_pii +from sentry_sdk.integrations._wsgi_common import _filter_headers +from sentry_sdk.utils import ContextVar, event_from_exception, transaction_from_function +from sentry_sdk.tracing import Span + +if MYPY: + from typing import Dict + from typing import Any + from typing import Optional + from typing import Callable + + from sentry_sdk._types import Event, Hint + + +_asgi_middleware_applied = ContextVar("sentry_asgi_middleware_applied") + + +def _capture_exception(hub, exc): + # type: (Hub, Any) -> None + + # Check client here as it might have been unset while streaming response + if hub.client is not None: + event, hint = event_from_exception( + exc, + client_options=hub.client.options, + mechanism={"type": "asgi", "handled": False}, + ) + hub.capture_event(event, hint=hint) + + +def _looks_like_asgi3(app): + # type: (Any) -> bool + """ + Try to figure out if an application object supports ASGI3. + + This is how uvicorn figures out the application version as well. + """ + if inspect.isclass(app): + return hasattr(app, "__await__") + elif inspect.isfunction(app): + return asyncio.iscoroutinefunction(app) + else: + call = getattr(app, "__call__", None) # noqa + return asyncio.iscoroutinefunction(call) + + +class SentryAsgiMiddleware: + __slots__ = ("app", "__call__") + + def __init__(self, app): + # type: (Any) -> None + self.app = app + + if _looks_like_asgi3(app): + self.__call__ = self._run_asgi3 # type: Callable[..., Any] + else: + self.__call__ = self._run_asgi2 + + def _run_asgi2(self, scope): + # type: (Any) -> Any + async def inner(receive, send): + # type: (Any, Any) -> Any + return await self._run_app(scope, lambda: self.app(scope)(receive, send)) + + return inner + + async def _run_asgi3(self, scope, receive, send): + # type: (Any, Any, Any) -> Any + return await self._run_app(scope, lambda: self.app(scope, receive, send)) + + async def _run_app(self, scope, callback): + # type: (Any, Any) -> Any + if _asgi_middleware_applied.get(False): + return await callback() + + _asgi_middleware_applied.set(True) + try: + hub = Hub(Hub.current) + with hub: + with hub.configure_scope() as sentry_scope: + sentry_scope.clear_breadcrumbs() + sentry_scope._name = "asgi" + processor = functools.partial( + self.event_processor, asgi_scope=scope + ) + sentry_scope.add_event_processor(processor) + + if scope["type"] in ("http", "websocket"): + span = Span.continue_from_headers(dict(scope["headers"])) + span.op = "{}.server".format(scope["type"]) + else: + span = Span() + span.op = "asgi.server" + + span.set_tag("asgi.type", scope["type"]) + span.transaction = "generic ASGI request" + + with hub.start_span(span) as span: + # XXX: Would be cool to have correct span status, but we + # would have to wrap send(). That is a bit hard to do with + # the current abstraction over ASGI 2/3. + try: + return await callback() + except Exception as exc: + _capture_exception(hub, exc) + raise exc from None + finally: + _asgi_middleware_applied.set(False) + + def event_processor(self, event, hint, asgi_scope): + # type: (Event, Hint, Any) -> Optional[Event] + request_info = event.get("request", {}) + + if asgi_scope["type"] in ("http", "websocket"): + request_info["url"] = self.get_url(asgi_scope) + request_info["method"] = asgi_scope["method"] + request_info["headers"] = _filter_headers(self.get_headers(asgi_scope)) + request_info["query_string"] = self.get_query(asgi_scope) + + if asgi_scope.get("client") and _should_send_default_pii(): + request_info["env"] = {"REMOTE_ADDR": asgi_scope["client"][0]} + + if asgi_scope.get("endpoint"): + # Webframeworks like Starlette mutate the ASGI env once routing is + # done, which is sometime after the request has started. If we have + # an endpoint, overwrite our path-based transaction name. + event["transaction"] = self.get_transaction(asgi_scope) + + event["request"] = request_info + + return event + + def get_url(self, scope): + # type: (Any) -> str + """ + Extract URL from the ASGI scope, without also including the querystring. + """ + scheme = scope.get("scheme", "http") + server = scope.get("server", None) + path = scope.get("root_path", "") + scope["path"] + + for key, value in scope["headers"]: + if key == b"host": + host_header = value.decode("latin-1") + return "%s://%s%s" % (scheme, host_header, path) + + if server is not None: + host, port = server + default_port = {"http": 80, "https": 443, "ws": 80, "wss": 443}[scheme] + if port != default_port: + return "%s://%s:%s%s" % (scheme, host, port, path) + return "%s://%s%s" % (scheme, host, path) + return path + + def get_query(self, scope): + # type: (Any) -> Any + """ + Extract querystring from the ASGI scope, in the format that the Sentry protocol expects. + """ + return urllib.parse.unquote(scope["query_string"].decode("latin-1")) + + def get_headers(self, scope): + # type: (Any) -> Dict[str, Any] + """ + Extract headers from the ASGI scope, in the format that the Sentry protocol expects. + """ + headers = {} # type: Dict[str, str] + for raw_key, raw_value in scope["headers"]: + key = raw_key.decode("latin-1") + value = raw_value.decode("latin-1") + if key in headers: + headers[key] = headers[key] + ", " + value + else: + headers[key] = value + return headers + + def get_transaction(self, scope): + # type: (Any) -> Optional[str] + """ + Return a transaction string to identify the routed endpoint. + """ + return transaction_from_function(scope["endpoint"]) diff --git a/sentry_sdk/integrations/atexit.py b/sentry_sdk/integrations/atexit.py old mode 100755 new mode 100644 index ecaa82b..3d0eca8 --- a/sentry_sdk/integrations/atexit.py +++ b/sentry_sdk/integrations/atexit.py @@ -17,6 +17,7 @@ def default_callback(pending, timeout): + # type: (int, int) -> None """This is the default shutdown callback that is set on the options. It prints out a message to stderr that informs the user that some events are still pending and the process is waiting for them to flush out. @@ -46,6 +47,7 @@ def setup_once(): # type: () -> None @atexit.register def _shutdown(): + # type: () -> None logger.debug("atexit: got shutdown signal") hub = Hub.main integration = hub.get_integration(AtexitIntegration) diff --git a/sentry_sdk/integrations/aws_lambda.py b/sentry_sdk/integrations/aws_lambda.py old mode 100755 new mode 100644 index c96f9ab..f1b5b38 --- a/sentry_sdk/integrations/aws_lambda.py +++ b/sentry_sdk/integrations/aws_lambda.py @@ -15,10 +15,19 @@ if MYPY: from typing import Any + from typing import TypeVar + from typing import Callable + from typing import Optional + + from sentry_sdk._types import EventProcessor, Event, Hint + + F = TypeVar("F", bound=Callable[..., Any]) def _wrap_handler(handler): + # type: (F) -> F def sentry_handler(event, context, *args, **kwargs): + # type: (Any, Any, *Any, **Any) -> Any hub = Hub.current integration = hub.get_integration(AwsLambdaIntegration) if integration is None: @@ -45,10 +54,11 @@ def sentry_handler(event, context, *args, **kwargs): hub.capture_event(event, hint=hint) reraise(*exc_info) - return sentry_handler + return sentry_handler # type: ignore def _drain_queue(): + # type: () -> None with capture_internal_exceptions(): hub = Hub.current integration = hub.get_integration(AwsLambdaIntegration) @@ -87,6 +97,7 @@ def setup_once(): old_handle_event_request = lambda_bootstrap.handle_event_request def sentry_handle_event_request(request_handler, *args, **kwargs): + # type: (Any, *Any, **Any) -> Any request_handler = _wrap_handler(request_handler) return old_handle_event_request(request_handler, *args, **kwargs) @@ -95,6 +106,7 @@ def sentry_handle_event_request(request_handler, *args, **kwargs): old_handle_http_request = lambda_bootstrap.handle_http_request def sentry_handle_http_request(request_handler, *args, **kwargs): + # type: (Any, *Any, **Any) -> Any request_handler = _wrap_handler(request_handler) return old_handle_http_request(request_handler, *args, **kwargs) @@ -106,6 +118,7 @@ def sentry_handle_http_request(request_handler, *args, **kwargs): old_to_json = lambda_bootstrap.to_json def sentry_to_json(*args, **kwargs): + # type: (*Any, **Any) -> Any _drain_queue() return old_to_json(*args, **kwargs) @@ -127,11 +140,13 @@ def sentry_handle_event_request( # type: ignore # even when the SDK is initialized inside of the handler def _wrap_post_function(f): + # type: (F) -> F def inner(*args, **kwargs): + # type: (*Any, **Any) -> Any _drain_queue() return f(*args, **kwargs) - return inner + return inner # type: ignore lambda_bootstrap.LambdaRuntimeClient.post_invocation_result = _wrap_post_function( lambda_bootstrap.LambdaRuntimeClient.post_invocation_result @@ -142,7 +157,9 @@ def inner(*args, **kwargs): def _make_request_event_processor(aws_event, aws_context): + # type: (Any, Any) -> EventProcessor def event_processor(event, hint): + # type: (Event, Hint) -> Optional[Event] extra = event.setdefault("extra", {}) extra["lambda"] = { "remaining_time_in_millis": aws_context.get_remaining_time_in_millis(), @@ -152,7 +169,7 @@ def event_processor(event, hint): "aws_request_id": aws_context.aws_request_id, } - request = event.setdefault("request", {}) + request = event.get("request", {}) if "httpMethod" in aws_event: request["method"] = aws_event["httpMethod"] @@ -181,12 +198,15 @@ def event_processor(event, hint): if ip is not None: user_info["ip_address"] = ip + event["request"] = request + return event return event_processor def _get_url(event, context): + # type: (Any, Any) -> str path = event.get("path", None) headers = event.get("headers", {}) host = headers.get("Host", None) diff --git a/sentry_sdk/integrations/beam.py b/sentry_sdk/integrations/beam.py new file mode 100644 index 0000000..7252746 --- /dev/null +++ b/sentry_sdk/integrations/beam.py @@ -0,0 +1,184 @@ +from __future__ import absolute_import + +import sys +import types +from functools import wraps + +from sentry_sdk.hub import Hub +from sentry_sdk._compat import reraise +from sentry_sdk.utils import capture_internal_exceptions, event_from_exception +from sentry_sdk.integrations import Integration +from sentry_sdk.integrations.logging import ignore_logger +from sentry_sdk._types import MYPY + +if MYPY: + from typing import Any + from typing import Iterator + from typing import TypeVar + from typing import Optional + from typing import Callable + + from sentry_sdk.client import Client + from sentry_sdk._types import ExcInfo + + T = TypeVar("T") + F = TypeVar("F", bound=Callable[..., Any]) + + +WRAPPED_FUNC = "_wrapped_{}_" +INSPECT_FUNC = "_inspect_{}" # Required format per apache_beam/transforms/core.py +USED_FUNC = "_sentry_used_" + + +class BeamIntegration(Integration): + identifier = "beam" + + @staticmethod + def setup_once(): + # type: () -> None + from apache_beam.transforms.core import DoFn, ParDo # type: ignore + + ignore_logger("root") + ignore_logger("bundle_processor.create") + + function_patches = ["process", "start_bundle", "finish_bundle", "setup"] + for func_name in function_patches: + setattr( + DoFn, + INSPECT_FUNC.format(func_name), + _wrap_inspect_call(DoFn, func_name), + ) + + old_init = ParDo.__init__ + + def sentry_init_pardo(self, fn, *args, **kwargs): + # type: (ParDo, Any, *Any, **Any) -> Any + # Do not monkey patch init twice + if not getattr(self, "_sentry_is_patched", False): + for func_name in function_patches: + if not hasattr(fn, func_name): + continue + wrapped_func = WRAPPED_FUNC.format(func_name) + + # Check to see if inspect is set and process is not + # to avoid monkey patching process twice. + # Check to see if function is part of object for + # backwards compatibility. + process_func = getattr(fn, func_name) + inspect_func = getattr(fn, INSPECT_FUNC.format(func_name)) + if not getattr(inspect_func, USED_FUNC, False) and not getattr( + process_func, USED_FUNC, False + ): + setattr(fn, wrapped_func, process_func) + setattr(fn, func_name, _wrap_task_call(process_func)) + + self._sentry_is_patched = True + old_init(self, fn, *args, **kwargs) + + ParDo.__init__ = sentry_init_pardo + + +def _wrap_inspect_call(cls, func_name): + # type: (Any, Any) -> Any + from apache_beam.typehints.decorators import getfullargspec # type: ignore + + if not hasattr(cls, func_name): + return None + + def _inspect(self): + # type: (Any) -> Any + """ + Inspect function overrides the way Beam gets argspec. + """ + wrapped_func = WRAPPED_FUNC.format(func_name) + if hasattr(self, wrapped_func): + process_func = getattr(self, wrapped_func) + else: + process_func = getattr(self, func_name) + setattr(self, func_name, _wrap_task_call(process_func)) + setattr(self, wrapped_func, process_func) + + # getfullargspec is deprecated in more recent beam versions and get_function_args_defaults + # (which uses Signatures internally) should be used instead. + try: + from apache_beam.transforms.core import get_function_args_defaults + + return get_function_args_defaults(process_func) + except ImportError: + return getfullargspec(process_func) + + setattr(_inspect, USED_FUNC, True) + return _inspect + + +def _wrap_task_call(func): + # type: (F) -> F + """ + Wrap task call with a try catch to get exceptions. + Pass the client on to raise_exception so it can get rebinded. + """ + client = Hub.current.client + + @wraps(func) + def _inner(*args, **kwargs): + # type: (*Any, **Any) -> Any + try: + gen = func(*args, **kwargs) + except Exception: + raise_exception(client) + + if not isinstance(gen, types.GeneratorType): + return gen + return _wrap_generator_call(gen, client) + + setattr(_inner, USED_FUNC, True) + return _inner # type: ignore + + +def _capture_exception(exc_info, hub): + # type: (ExcInfo, Hub) -> None + """ + Send Beam exception to Sentry. + """ + integration = hub.get_integration(BeamIntegration) + if integration is None: + return + + client = hub.client + if client is None: + return + + event, hint = event_from_exception( + exc_info, + client_options=client.options, + mechanism={"type": "beam", "handled": False}, + ) + hub.capture_event(event, hint=hint) + + +def raise_exception(client): + # type: (Optional[Client]) -> None + """ + Raise an exception. If the client is not in the hub, rebind it. + """ + hub = Hub.current + if hub.client is None: + hub.bind_client(client) + exc_info = sys.exc_info() + with capture_internal_exceptions(): + _capture_exception(exc_info, hub) + reraise(*exc_info) + + +def _wrap_generator_call(gen, client): + # type: (Iterator[T], Optional[Client]) -> Iterator[T] + """ + Wrap the generator to handle any failures. + """ + while True: + try: + yield next(gen) + except StopIteration: + break + except Exception: + raise_exception(client) diff --git a/sentry_sdk/integrations/bottle.py b/sentry_sdk/integrations/bottle.py old mode 100755 new mode 100644 index b008a19..93ca96e --- a/sentry_sdk/integrations/bottle.py +++ b/sentry_sdk/integrations/bottle.py @@ -13,7 +13,6 @@ from sentry_sdk._types import MYPY if MYPY: - from sentry_sdk.integrations.wsgi import _ScopedResponse from typing import Any from typing import Dict @@ -21,12 +20,9 @@ from typing import Optional from bottle import FileUpload, FormsDict, LocalRequest # type: ignore -from bottle import ( - Bottle, - Route, - request as bottle_request, - HTTPResponse, -) # type: ignore + from sentry_sdk._types import EventProcessor + +from bottle import Bottle, Route, request as bottle_request, HTTPResponse class BottleIntegration(Integration): @@ -52,7 +48,7 @@ def setup_once(): old_app = Bottle.__call__ def sentry_patched_wsgi_app(self, environ, start_response): - # type: (Any, Dict[str, str], Callable) -> _ScopedResponse + # type: (Any, Dict[str, str], Callable[..., Any]) -> _ScopedResponse hub = Hub.current integration = hub.get_integration(BottleIntegration) @@ -63,12 +59,13 @@ def sentry_patched_wsgi_app(self, environ, start_response): environ, start_response ) - Bottle.__call__ = sentry_patched_wsgi_app # type: ignore + Bottle.__call__ = sentry_patched_wsgi_app # monkey patch method Bottle._handle old_handle = Bottle._handle def _patched_handle(self, environ): + # type: (Bottle, Dict[str, Any]) -> Any hub = Hub.current integration = hub.get_integration(BottleIntegration) if integration is None: @@ -95,6 +92,7 @@ def _patched_handle(self, environ): old_make_callback = Route._make_callback def patched_make_callback(self, *args, **kwargs): + # type: (Route, *object, **object) -> Any hub = Hub.current integration = hub.get_integration(BottleIntegration) prepared_callback = old_make_callback(self, *args, **kwargs) @@ -105,20 +103,19 @@ def patched_make_callback(self, *args, **kwargs): client = hub.client # type: Any def wrapped_callback(*args, **kwargs): - def capture_exception(exception): - event, hint = event_from_exception( - exception, - client_options=client.options, - mechanism={"type": "bottle", "handled": False}, - ) - hub.capture_event(event, hint=hint) + # type: (*object, **object) -> Any try: res = prepared_callback(*args, **kwargs) except HTTPResponse: raise except Exception as exception: - capture_exception(exception) + event, hint = event_from_exception( + exception, + client_options=client.options, + mechanism={"type": "bottle", "handled": False}, + ) + hub.capture_event(event, hint=hint) raise exception return res @@ -160,7 +157,7 @@ def size_of_file(self, file): def _make_request_event_processor(app, request, integration): - # type: (Bottle, LocalRequest, BottleIntegration) -> Callable + # type: (Bottle, LocalRequest, BottleIntegration) -> EventProcessor def inner(event, hint): # type: (Dict[str, Any], Dict[str, Any]) -> Dict[str, Any] @@ -170,7 +167,7 @@ def inner(event, hint): request.route.callback ) elif integration.transaction_style == "url": - event["transaction"] = request.route.rule # type: ignore + event["transaction"] = request.route.rule except Exception: pass diff --git a/sentry_sdk/integrations/celery.py b/sentry_sdk/integrations/celery.py old mode 100755 new mode 100644 index 255e60e..da0ee5c --- a/sentry_sdk/integrations/celery.py +++ b/sentry_sdk/integrations/celery.py @@ -1,5 +1,6 @@ from __future__ import absolute_import +import functools import sys from celery.exceptions import ( # type: ignore @@ -15,6 +16,17 @@ from sentry_sdk._compat import reraise from sentry_sdk.integrations import Integration from sentry_sdk.integrations.logging import ignore_logger +from sentry_sdk._types import MYPY + +if MYPY: + from typing import Any + from typing import TypeVar + from typing import Callable + from typing import Optional + + from sentry_sdk._types import EventProcessor, Event, Hint, ExcInfo + + F = TypeVar("F", bound=Callable[..., Any]) CELERY_CONTROL_FLOW_EXCEPTIONS = (Retry, Ignore, Reject) @@ -35,6 +47,7 @@ def setup_once(): old_build_tracer = trace.build_tracer def sentry_build_tracer(name, task, *args, **kwargs): + # type: (Any, Any, *Any, **Any) -> Any if not getattr(task, "_sentry_is_patched", False): # Need to patch both methods because older celery sometimes # short-circuits to task.run if it thinks it's safe. @@ -57,10 +70,18 @@ def sentry_build_tracer(name, task, *args, **kwargs): # Meaning that every task's breadcrumbs are full of stuff like "Task # raised unexpected ". ignore_logger("celery.worker.job") + ignore_logger("celery.app.trace") + + # This is stdout/err redirected to a logger, can't deal with this + # (need event_level=logging.WARN to reproduce) + ignore_logger("celery.redirected") def _wrap_apply_async(task, f): + # type: (Any, F) -> F + @functools.wraps(f) def apply_async(*args, **kwargs): + # type: (*Any, **Any) -> Any hub = Hub.current integration = hub.get_integration(CeleryIntegration) if integration is not None and integration.propagate_traces: @@ -71,19 +92,27 @@ def apply_async(*args, **kwargs): headers[key] = value if headers is not None: kwargs["headers"] = headers - return f(*args, **kwargs) - return apply_async + with hub.start_span(op="celery.submit", description=task.name): + return f(*args, **kwargs) + else: + return f(*args, **kwargs) + + return apply_async # type: ignore def _wrap_tracer(task, f): + # type: (Any, F) -> F + # Need to wrap tracer for pushing the scope before prerun is sent, and # popping it after postrun is sent. # # This is the reason we don't use signals for hooking in the first place. # Also because in Celery 3, signal dispatch returns early if one handler # crashes. + @functools.wraps(f) def _inner(*args, **kwargs): + # type: (*Any, **Any) -> Any hub = Hub.current if hub.get_integration(CeleryIntegration) is None: return f(*args, **kwargs) @@ -94,23 +123,35 @@ def _inner(*args, **kwargs): scope.add_event_processor(_make_event_processor(task, *args, **kwargs)) span = Span.continue_from_headers(args[3].get("headers") or {}) + span.op = "celery.task" span.transaction = "unknown celery task" + # Could possibly use a better hook than this one + span.set_status("ok") + with capture_internal_exceptions(): # Celery task objects are not a thing to be trusted. Even # something such as attribute access can fail. span.transaction = task.name - with hub.span(span): + with hub.start_span(span): return f(*args, **kwargs) - return _inner + return _inner # type: ignore def _wrap_task_call(task, f): + # type: (Any, F) -> F + # Need to wrap task call because the exception is caught before we get to # see it. Also celery's reported stacktrace is untrustworthy. + + # functools.wraps is important here because celery-once looks at this + # method's name. + # https://github.com/getsentry/sentry-python/issues/421 + @functools.wraps(f) def _inner(*args, **kwargs): + # type: (*Any, **Any) -> Any try: return f(*args, **kwargs) except Exception: @@ -119,11 +160,14 @@ def _inner(*args, **kwargs): _capture_exception(task, exc_info) reraise(*exc_info) - return _inner + return _inner # type: ignore def _make_event_processor(task, uuid, args, kwargs, request=None): + # type: (Any, Any, Any, Any, Optional[Any]) -> EventProcessor def event_processor(event, hint): + # type: (Event, Hint) -> Optional[Event] + with capture_internal_exceptions(): extra = event.setdefault("extra", {}) extra["celery-job"] = { @@ -147,25 +191,44 @@ def event_processor(event, hint): def _capture_exception(task, exc_info): + # type: (Any, ExcInfo) -> None hub = Hub.current if hub.get_integration(CeleryIntegration) is None: return if isinstance(exc_info[1], CELERY_CONTROL_FLOW_EXCEPTIONS): + # ??? Doesn't map to anything + _set_status(hub, "aborted") return + + _set_status(hub, "internal_error") + if hasattr(task, "throws") and isinstance(exc_info[1], task.throws): return + # If an integration is there, a client has to be there. + client = hub.client # type: Any + event, hint = event_from_exception( exc_info, - client_options=hub.client.options, + client_options=client.options, mechanism={"type": "celery", "handled": False}, ) hub.capture_event(event, hint=hint) +def _set_status(hub, status): + # type: (Hub, str) -> None + with capture_internal_exceptions(): + with hub.configure_scope() as scope: + if scope.span is not None: + scope.span.set_status(status) + + def _patch_worker_exit(): + # type: () -> None + # Need to flush queue before worker shutdown because a crashing worker will # call os._exit from billiard.pool import Worker # type: ignore @@ -173,6 +236,7 @@ def _patch_worker_exit(): old_workloop = Worker.workloop def sentry_workloop(*args, **kwargs): + # type: (*Any, **Any) -> Any try: return old_workloop(*args, **kwargs) finally: diff --git a/sentry_sdk/integrations/dedupe.py b/sentry_sdk/integrations/dedupe.py old mode 100755 new mode 100644 diff --git a/sentry_sdk/integrations/django/__init__.py b/sentry_sdk/integrations/django/__init__.py old mode 100755 new mode 100644 index 45f76f7..0d32e1c --- a/sentry_sdk/integrations/django/__init__.py +++ b/sentry_sdk/integrations/django/__init__.py @@ -5,10 +5,11 @@ import threading import weakref -from django import VERSION as DJANGO_VERSION # type: ignore -from django.core import signals # type: ignore +from django import VERSION as DJANGO_VERSION +from django.core import signals from sentry_sdk._types import MYPY +from sentry_sdk.utils import HAS_REAL_CONTEXTVARS, logger if MYPY: from typing import Any @@ -16,20 +17,21 @@ from typing import Dict from typing import Optional from typing import Union + from typing import List - from django.core.handlers.wsgi import WSGIRequest # type: ignore - from django.http.response import HttpResponse # type: ignore - from django.http.request import QueryDict # type: ignore - from django.utils.datastructures import MultiValueDict # type: ignore + from django.core.handlers.wsgi import WSGIRequest + from django.http.response import HttpResponse + from django.http.request import QueryDict + from django.utils.datastructures import MultiValueDict from sentry_sdk.integrations.wsgi import _ScopedResponse - from sentry_sdk._types import Event, Hint + from sentry_sdk._types import Event, Hint, EventProcessor, NotImplementedType try: - from django.urls import resolve # type: ignore + from django.urls import resolve except ImportError: - from django.core.urlresolvers import resolve # type: ignore + from django.core.urlresolvers import resolve from sentry_sdk import Hub from sentry_sdk.hub import _should_send_default_pii @@ -46,9 +48,9 @@ from sentry_sdk.integrations.logging import ignore_logger from sentry_sdk.integrations.wsgi import SentryWsgiMiddleware from sentry_sdk.integrations._wsgi_common import RequestExtractor -from sentry_sdk.integrations._sql_common import format_sql from sentry_sdk.integrations.django.transactions import LEGACY_RESOLVER from sentry_sdk.integrations.django.templates import get_template_frame_from_exception +from sentry_sdk.integrations.django.middleware import patch_django_middlewares if DJANGO_VERSION < (1, 10): @@ -69,9 +71,10 @@ class DjangoIntegration(Integration): identifier = "django" transaction_style = None + middleware_spans = None - def __init__(self, transaction_style="url"): - # type: (str) -> None + def __init__(self, transaction_style="url", middleware_spans=True): + # type: (str, bool) -> None TRANSACTION_STYLE_VALUES = ("function_name", "url") if transaction_style not in TRANSACTION_STYLE_VALUES: raise ValueError( @@ -79,6 +82,7 @@ def __init__(self, transaction_style="url"): % (transaction_style, TRANSACTION_STYLE_VALUES) ) self.transaction_style = transaction_style + self.middleware_spans = middleware_spans @staticmethod def setup_once(): @@ -95,19 +99,19 @@ def setup_once(): old_app = WSGIHandler.__call__ def sentry_patched_wsgi_handler(self, environ, start_response): - # type: (Any, Dict[str, str], Callable) -> _ScopedResponse + # type: (Any, Dict[str, str], Callable[..., Any]) -> _ScopedResponse if Hub.current.get_integration(DjangoIntegration) is None: return old_app(self, environ, start_response) - return SentryWsgiMiddleware(lambda *a, **kw: old_app(self, *a, **kw))( - environ, start_response - ) + bound_old_app = old_app.__get__(self, WSGIHandler) + + return SentryWsgiMiddleware(bound_old_app)(environ, start_response) WSGIHandler.__call__ = sentry_patched_wsgi_handler # patch get_response, because at that point we have the Django request # object - from django.core.handlers.base import BaseHandler # type: ignore + from django.core.handlers.base import BaseHandler old_get_response = BaseHandler.get_response @@ -184,6 +188,7 @@ def process_django_templates(event, hint): @add_global_repr_processor def _django_queryset_repr(value, hint): + # type: (Any, Dict[str, Any]) -> Union[NotImplementedType, str] try: # Django 1.6 can fail to import `QuerySet` when Django settings # have not yet been initialized. @@ -191,7 +196,7 @@ def _django_queryset_repr(value, hint): # If we fail to import, return `NotImplemented`. It's at least # unlikely that we have a query set in `value` when importing # `QuerySet` fails. - from django.db.models.query import QuerySet # type: ignore + from django.db.models.query import QuerySet except Exception: return NotImplemented @@ -209,12 +214,16 @@ def _django_queryset_repr(value, hint): id(value), ) + _patch_channels() + patch_django_middlewares() + _DRF_PATCHED = False _DRF_PATCH_LOCK = threading.Lock() def _patch_drf(): + # type: () -> None """ Patch Django Rest Framework for more/better request data. DRF's request type is a wrapper around Django's request type. The attribute we're @@ -257,6 +266,7 @@ def _patch_drf(): old_drf_initial = APIView.initial def sentry_patched_drf_initial(self, request, *args, **kwargs): + # type: (APIView, Any, *Any, **Any) -> Any with capture_internal_exceptions(): request._request._sentry_drf_request_backref = weakref.ref( request @@ -267,8 +277,45 @@ def sentry_patched_drf_initial(self, request, *args, **kwargs): APIView.initial = sentry_patched_drf_initial +def _patch_channels(): + # type: () -> None + try: + from channels.http import AsgiHandler # type: ignore + except ImportError: + return + + if not HAS_REAL_CONTEXTVARS: + # We better have contextvars or we're going to leak state between + # requests. + # + # We cannot hard-raise here because channels may not be used at all in + # the current process. + logger.warning( + "We detected that you are using Django channels 2.0. To get proper " + "instrumentation for ASGI requests, the Sentry SDK requires " + "Python 3.7+ or the aiocontextvars package from PyPI." + ) + + from sentry_sdk.integrations.asgi import SentryAsgiMiddleware + + old_app = AsgiHandler.__call__ + + def sentry_patched_asgi_handler(self, receive, send): + # type: (AsgiHandler, Any, Any) -> Any + if Hub.current.get_integration(DjangoIntegration) is None: + return old_app(receive, send) + + middleware = SentryAsgiMiddleware( + lambda _scope: old_app.__get__(self, AsgiHandler) + ) + + return middleware(self.scope)(receive, send) + + AsgiHandler.__call__ = sentry_patched_asgi_handler + + def _make_event_processor(weak_request, integration): - # type: (Callable[[], WSGIRequest], DjangoIntegration) -> Callable + # type: (Callable[[], WSGIRequest], DjangoIntegration) -> EventProcessor def event_processor(event, hint): # type: (Dict[str, Any], Dict[str, Any]) -> Dict[str, Any] # if the request is gone we are fine not logging the data from @@ -336,9 +383,11 @@ def files(self): return self.request.FILES def size_of_file(self, file): + # type: (Any) -> int return file.size def parsed_body(self): + # type: () -> Optional[Dict[str, Any]] try: return self.request.data except AttributeError: @@ -374,9 +423,9 @@ def install_sql_hook(): # type: () -> None """If installed this causes Django's queries to be captured.""" try: - from django.db.backends.utils import CursorWrapper # type: ignore + from django.db.backends.utils import CursorWrapper except ImportError: - from django.db.backends.util import CursorWrapper # type: ignore + from django.db.backends.util import CursorWrapper try: real_execute = CursorWrapper.execute @@ -386,24 +435,24 @@ def install_sql_hook(): return def execute(self, sql, params=None): + # type: (CursorWrapper, Any, Optional[Any]) -> Any hub = Hub.current if hub.get_integration(DjangoIntegration) is None: return real_execute(self, sql, params) with record_sql_queries( - hub, [format_sql(sql, params, self.cursor)], label="Django: " + hub, self.cursor, sql, params, paramstyle="format", executemany=False ): return real_execute(self, sql, params) def executemany(self, sql, param_list): + # type: (CursorWrapper, Any, List[Any]) -> Any hub = Hub.current if hub.get_integration(DjangoIntegration) is None: return real_executemany(self, sql, param_list) with record_sql_queries( - hub, - [format_sql(sql, params, self.cursor) for params in param_list], - label="Django: ", + hub, self.cursor, sql, param_list, paramstyle="format", executemany=True ): return real_executemany(self, sql, param_list) diff --git a/sentry_sdk/integrations/django/middleware.py b/sentry_sdk/integrations/django/middleware.py new file mode 100644 index 0000000..edbeccb --- /dev/null +++ b/sentry_sdk/integrations/django/middleware.py @@ -0,0 +1,136 @@ +""" +Create spans from Django middleware invocations +""" + +from functools import wraps + +from django import VERSION as DJANGO_VERSION + +from sentry_sdk import Hub +from sentry_sdk.utils import ( + ContextVar, + transaction_from_function, + capture_internal_exceptions, +) + +from sentry_sdk._types import MYPY + +if MYPY: + from typing import Any + from typing import Callable + from typing import TypeVar + + F = TypeVar("F", bound=Callable[..., Any]) + +_import_string_should_wrap_middleware = ContextVar( + "import_string_should_wrap_middleware" +) + +if DJANGO_VERSION < (1, 7): + import_string_name = "import_by_path" +else: + import_string_name = "import_string" + + +def patch_django_middlewares(): + # type: () -> None + from django.core.handlers import base + + old_import_string = getattr(base, import_string_name) + + def sentry_patched_import_string(dotted_path): + # type: (str) -> Any + rv = old_import_string(dotted_path) + + if _import_string_should_wrap_middleware.get(None): + rv = _wrap_middleware(rv, dotted_path) + + return rv + + setattr(base, import_string_name, sentry_patched_import_string) + + old_load_middleware = base.BaseHandler.load_middleware + + def sentry_patched_load_middleware(self): + # type: (base.BaseHandler) -> Any + _import_string_should_wrap_middleware.set(True) + try: + return old_load_middleware(self) + finally: + _import_string_should_wrap_middleware.set(False) + + base.BaseHandler.load_middleware = sentry_patched_load_middleware + + +def _wrap_middleware(middleware, middleware_name): + # type: (Any, str) -> Any + from sentry_sdk.integrations.django import DjangoIntegration + + def _get_wrapped_method(old_method): + # type: (F) -> F + with capture_internal_exceptions(): + + def sentry_wrapped_method(*args, **kwargs): + # type: (*Any, **Any) -> Any + hub = Hub.current + integration = hub.get_integration(DjangoIntegration) + if integration is None or not integration.middleware_spans: + return old_method(*args, **kwargs) + + function_name = transaction_from_function(old_method) + + description = middleware_name + function_basename = getattr(old_method, "__name__", None) + if function_basename: + description = "{}.{}".format(description, function_basename) + + with hub.start_span( + op="django.middleware", description=description + ) as span: + span.set_tag("django.function_name", function_name) + span.set_tag("django.middleware_name", middleware_name) + return old_method(*args, **kwargs) + + try: + # fails for __call__ of function on Python 2 (see py2.7-django-1.11) + return wraps(old_method)(sentry_wrapped_method) # type: ignore + except Exception: + return sentry_wrapped_method # type: ignore + + return old_method + + class SentryWrappingMiddleware(object): + def __init__(self, *args, **kwargs): + # type: (*Any, **Any) -> None + self._inner = middleware(*args, **kwargs) + self._call_method = None + + # We need correct behavior for `hasattr()`, which we can only determine + # when we have an instance of the middleware we're wrapping. + def __getattr__(self, method_name): + # type: (str) -> Any + if method_name not in ( + "process_request", + "process_view", + "process_template_response", + "process_response", + "process_exception", + ): + raise AttributeError() + + old_method = getattr(self._inner, method_name) + rv = _get_wrapped_method(old_method) + self.__dict__[method_name] = rv + return rv + + def __call__(self, *args, **kwargs): + # type: (*Any, **Any) -> Any + f = self._call_method + if f is None: + self._call_method = f = _get_wrapped_method(self._inner.__call__) + return f(*args, **kwargs) + + if hasattr(middleware, "__name__"): + SentryWrappingMiddleware.__name__ = middleware.__name__ + + return SentryWrappingMiddleware diff --git a/sentry_sdk/integrations/django/templates.py b/sentry_sdk/integrations/django/templates.py old mode 100755 new mode 100644 index 2f99976..2285644 --- a/sentry_sdk/integrations/django/templates.py +++ b/sentry_sdk/integrations/django/templates.py @@ -1,4 +1,4 @@ -from django.template import TemplateSyntaxError # type: ignore +from django.template import TemplateSyntaxError from sentry_sdk._types import MYPY @@ -6,13 +6,15 @@ from typing import Any from typing import Dict from typing import Optional + from typing import Iterator + from typing import Tuple try: # support Django 1.9 - from django.template.base import Origin # type: ignore + from django.template.base import Origin except ImportError: # backward compatibility - from django.template.loader import LoaderOrigin as Origin # type: ignore + from django.template.loader import LoaderOrigin as Origin def get_template_frame_from_exception(exc_value): @@ -33,7 +35,7 @@ def get_template_frame_from_exception(exc_value): if isinstance(exc_value, TemplateSyntaxError) and hasattr(exc_value, "source"): source = exc_value.source if isinstance(source, (tuple, list)) and isinstance(source[0], Origin): - return _get_template_frame_from_source(source) + return _get_template_frame_from_source(source) # type: ignore return None @@ -71,6 +73,7 @@ def _get_template_frame_from_debug(debug): def _linebreak_iter(template_source): + # type: (str) -> Iterator[int] yield 0 p = template_source.find("\n") while p >= 0: @@ -79,6 +82,7 @@ def _linebreak_iter(template_source): def _get_template_frame_from_source(source): + # type: (Tuple[Origin, Tuple[int, int]]) -> Optional[Dict[str, Any]] if not source: return None diff --git a/sentry_sdk/integrations/django/transactions.py b/sentry_sdk/integrations/django/transactions.py old mode 100755 new mode 100644 index 5e69532..a42328c --- a/sentry_sdk/integrations/django/transactions.py +++ b/sentry_sdk/integrations/django/transactions.py @@ -10,19 +10,19 @@ from sentry_sdk._types import MYPY if MYPY: - from django.urls.resolvers import URLResolver # type: ignore + from django.urls.resolvers import URLResolver from typing import Dict from typing import List from typing import Optional - from django.urls.resolvers import URLPattern # type: ignore + from django.urls.resolvers import URLPattern from typing import Tuple from typing import Union from re import Pattern # type: ignore try: - from django.urls import get_resolver # type: ignore + from django.urls import get_resolver except ImportError: - from django.core.urlresolvers import get_resolver # type: ignore + from django.core.urlresolvers import get_resolver def get_regex(resolver_or_pattern): diff --git a/sentry_sdk/integrations/excepthook.py b/sentry_sdk/integrations/excepthook.py old mode 100755 new mode 100644 index 7791de3..294a94b --- a/sentry_sdk/integrations/excepthook.py +++ b/sentry_sdk/integrations/excepthook.py @@ -9,6 +9,20 @@ if MYPY: from typing import Callable from typing import Any + from typing import Type + + from types import TracebackType + + from mypy_extensions import Arg + + Excepthook = Callable[ + [ + Arg(Type[BaseException], "type_"), + Arg(BaseException, "value"), + Arg(TracebackType, "traceback"), + ], + None, + ] class ExcepthookIntegration(Integration): @@ -33,8 +47,9 @@ def setup_once(): def _make_excepthook(old_excepthook): - # type: (Callable) -> Callable - def sentry_sdk_excepthook(exctype, value, traceback): + # type: (Excepthook) -> Excepthook + def sentry_sdk_excepthook(type_, value, traceback): + # type: (Type[BaseException], BaseException, TracebackType) -> None hub = Hub.current integration = hub.get_integration(ExcepthookIntegration) @@ -44,13 +59,13 @@ def sentry_sdk_excepthook(exctype, value, traceback): with capture_internal_exceptions(): event, hint = event_from_exception( - (exctype, value, traceback), + (type_, value, traceback), client_options=client.options, mechanism={"type": "excepthook", "handled": False}, ) hub.capture_event(event, hint=hint) - return old_excepthook(exctype, value, traceback) + return old_excepthook(type_, value, traceback) return sentry_sdk_excepthook diff --git a/sentry_sdk/integrations/falcon.py b/sentry_sdk/integrations/falcon.py old mode 100755 new mode 100644 index 06dbb1d..bf644b9 --- a/sentry_sdk/integrations/falcon.py +++ b/sentry_sdk/integrations/falcon.py @@ -12,24 +12,32 @@ if MYPY: from typing import Any - from typing import Callable from typing import Dict + from typing import Optional + + from sentry_sdk._types import EventProcessor class FalconRequestExtractor(RequestExtractor): def env(self): + # type: () -> Dict[str, Any] return self.request.env def cookies(self): + # type: () -> Dict[str, Any] return self.request.cookies def form(self): + # type: () -> None return None # No such concept in Falcon def files(self): + # type: () -> None return None # No such concept in Falcon def raw_data(self): + # type: () -> Optional[str] + # As request data can only be read once we won't make this available # to Sentry. Just send back a dummy string in case there was a # content length. @@ -41,6 +49,7 @@ def raw_data(self): return None def json(self): + # type: () -> Optional[Dict[str, Any]] try: return self.request.media except falcon.errors.HTTPBadRequest: @@ -55,6 +64,7 @@ class SentryFalconMiddleware(object): """Captures exceptions in Falcon requests and send to Sentry""" def process_request(self, req, resp, *args, **kwargs): + # type: (Any, Any, *Any, **Any) -> None hub = Hub.current integration = hub.get_integration(FalconIntegration) if integration is None: @@ -89,9 +99,11 @@ def setup_once(): def _patch_wsgi_app(): + # type: () -> None original_wsgi_app = falcon.API.__call__ def sentry_patched_wsgi_app(self, env, start_response): + # type: (falcon.API, Any, Any) -> Any hub = Hub.current integration = hub.get_integration(FalconIntegration) if integration is None: @@ -107,9 +119,11 @@ def sentry_patched_wsgi_app(self, env, start_response): def _patch_handle_exception(): + # type: () -> None original_handle_exception = falcon.API._handle_exception def sentry_patched_handle_exception(self, *args): + # type: (falcon.API, *Any) -> Any # NOTE(jmagnusson): falcon 2.0 changed falcon.API._handle_exception # method signature from `(ex, req, resp, params)` to # `(req, resp, ex, params)` @@ -140,11 +154,13 @@ def sentry_patched_handle_exception(self, *args): def _patch_prepare_middleware(): + # type: () -> None original_prepare_middleware = falcon.api_helpers.prepare_middleware def sentry_patched_prepare_middleware( middleware=None, independent_middleware=False ): + # type: (Any, Any) -> Any hub = Hub.current integration = hub.get_integration(FalconIntegration) if integration is not None: @@ -155,11 +171,12 @@ def sentry_patched_prepare_middleware( def _is_falcon_http_error(ex): + # type: (BaseException) -> bool return isinstance(ex, (falcon.HTTPError, falcon.http_status.HTTPStatus)) def _make_request_event_processor(req, integration): - # type: (falcon.Request, FalconIntegration) -> Callable + # type: (falcon.Request, FalconIntegration) -> EventProcessor def inner(event, hint): # type: (Dict[str, Any], Dict[str, Any]) -> Dict[str, Any] diff --git a/sentry_sdk/integrations/flask.py b/sentry_sdk/integrations/flask.py old mode 100755 new mode 100644 index 886faa4..8f2612e --- a/sentry_sdk/integrations/flask.py +++ b/sentry_sdk/integrations/flask.py @@ -11,7 +11,6 @@ from sentry_sdk._types import MYPY if MYPY: - from sentry_sdk.integrations.wsgi import _ScopedResponse from typing import Any from typing import Dict @@ -21,6 +20,8 @@ from typing import Union from typing import Callable + from sentry_sdk._types import EventProcessor + try: import flask_login # type: ignore except ImportError: @@ -61,7 +62,7 @@ def setup_once(): old_app = Flask.__call__ def sentry_patched_wsgi_app(self, environ, start_response): - # type: (Any, Dict[str, str], Callable) -> _ScopedResponse + # type: (Any, Dict[str, str], Callable[..., Any]) -> _ScopedResponse if Hub.current.get_integration(FlaskIntegration) is None: return old_app(self, environ, start_response) @@ -106,18 +107,17 @@ def _request_started(sender, **kwargs): # Rely on WSGI middleware to start a trace try: if integration.transaction_style == "endpoint": - scope.transaction = request.url_rule.endpoint # type: ignore + scope.transaction = request.url_rule.endpoint elif integration.transaction_style == "url": - scope.transaction = request.url_rule.rule # type: ignore + scope.transaction = request.url_rule.rule except Exception: pass weak_request = weakref.ref(request) - scope.add_event_processor( - _make_request_event_processor( # type: ignore - app, weak_request, integration - ) + evt_processor = _make_request_event_processor( + app, weak_request, integration # type: ignore ) + scope.add_event_processor(evt_processor) class FlaskRequestExtractor(RequestExtractor): @@ -126,25 +126,27 @@ def env(self): return self.request.environ def cookies(self): - # type: () -> ImmutableTypeConversionDict + # type: () -> ImmutableTypeConversionDict[Any, Any] return self.request.cookies def raw_data(self): # type: () -> bytes - return self.request.data + return self.request.get_data() def form(self): - # type: () -> ImmutableMultiDict + # type: () -> ImmutableMultiDict[str, Any] return self.request.form def files(self): - # type: () -> ImmutableMultiDict + # type: () -> ImmutableMultiDict[str, Any] return self.request.files def is_json(self): + # type: () -> bool return self.request.is_json def json(self): + # type: () -> Any return self.request.get_json() def size_of_file(self, file): @@ -153,7 +155,7 @@ def size_of_file(self, file): def _make_request_event_processor(app, weak_request, integration): - # type: (Flask, Callable[[], Request], FlaskIntegration) -> Callable + # type: (Flask, Callable[[], Request], FlaskIntegration) -> EventProcessor def inner(event, hint): # type: (Dict[str, Any], Dict[str, Any]) -> Dict[str, Any] request = weak_request() diff --git a/sentry_sdk/integrations/gnu_backtrace.py b/sentry_sdk/integrations/gnu_backtrace.py old mode 100755 new mode 100644 index 6671de9..e0ec110 --- a/sentry_sdk/integrations/gnu_backtrace.py +++ b/sentry_sdk/integrations/gnu_backtrace.py @@ -42,6 +42,7 @@ def setup_once(): # type: () -> None @add_global_event_processor def process_gnu_backtrace(event, hint): + # type: (Dict[str, Any], Dict[str, Any]) -> Dict[str, Any] with capture_internal_exceptions(): return _process_gnu_backtrace(event, hint) diff --git a/sentry_sdk/integrations/logging.py b/sentry_sdk/integrations/logging.py old mode 100755 new mode 100644 index 9e76c10..6b37c8b --- a/sentry_sdk/integrations/logging.py +++ b/sentry_sdk/integrations/logging.py @@ -19,6 +19,7 @@ from logging import LogRecord from typing import Any from typing import Dict + from typing import Optional DEFAULT_LEVEL = logging.INFO DEFAULT_EVENT_LEVEL = logging.ERROR @@ -26,12 +27,16 @@ _IGNORED_LOGGERS = set(["sentry_sdk.errors"]) -def ignore_logger(name): - # type: (str) -> None +def ignore_logger( + name # type: str +): + # type: (...) -> None """This disables recording (both in breadcrumbs and as events) calls to a logger of a specific name. Among other uses, many of our integrations use this to prevent their actions being recorded as breadcrumbs. Exposed to users as a way to quiet spammy loggers. + + :param name: The name of the logger to ignore (same string you would pass to ``logging.getLogger``). """ _IGNORED_LOGGERS.add(name) @@ -40,7 +45,7 @@ class LoggingIntegration(Integration): identifier = "logging" def __init__(self, level=DEFAULT_LEVEL, event_level=DEFAULT_EVENT_LEVEL): - # type: (int, int) -> None + # type: (Optional[int], Optional[int]) -> None self._handler = None self._breadcrumb_handler = None @@ -95,7 +100,7 @@ def _breadcrumb_from_record(record): "level": _logging_to_event_level(record.levelname), "category": record.name, "message": record.message, - "timestamp": datetime.datetime.fromtimestamp(record.created), + "timestamp": datetime.datetime.utcfromtimestamp(record.created), "data": _extra_from_record(record), } @@ -130,6 +135,7 @@ def _logging_to_event_level(levelname): "tags", "thread", "threadName", + "stack_info", ) ) @@ -139,11 +145,18 @@ def _extra_from_record(record): return { k: v for k, v in iteritems(vars(record)) - if k not in COMMON_RECORD_ATTRS and not k.startswith("_") + if k not in COMMON_RECORD_ATTRS + and (not isinstance(k, str) or not k.startswith("_")) } class EventHandler(logging.Handler, object): + """ + A logging handler that emits Sentry events for each log record + + Note that you do not have to use this class if the logging integration is enabled, which it is by default. + """ + def emit(self, record): # type: (LogRecord) -> Any with capture_internal_exceptions(): @@ -202,6 +215,12 @@ def _emit(self, record): class BreadcrumbHandler(logging.Handler, object): + """ + A logging handler that records breadcrumbs for each log record. + + Note that you do not have to use this class if the logging integration is enabled, which it is by default. + """ + def emit(self, record): # type: (LogRecord) -> Any with capture_internal_exceptions(): diff --git a/sentry_sdk/integrations/modules.py b/sentry_sdk/integrations/modules.py old mode 100755 new mode 100644 index f0238be..3d78cb8 --- a/sentry_sdk/integrations/modules.py +++ b/sentry_sdk/integrations/modules.py @@ -46,6 +46,11 @@ def setup_once(): @add_global_event_processor def processor(event, hint): # type: (Event, Any) -> Dict[str, Any] - if Hub.current.get_integration(ModulesIntegration) is not None: - event["modules"] = dict(_get_installed_modules()) + if event.get("type") == "transaction": + return event + + if Hub.current.get_integration(ModulesIntegration) is None: + return event + + event["modules"] = _get_installed_modules() return event diff --git a/sentry_sdk/integrations/pyramid.py b/sentry_sdk/integrations/pyramid.py old mode 100755 new mode 100644 index 4626db6..8e0cea1 --- a/sentry_sdk/integrations/pyramid.py +++ b/sentry_sdk/integrations/pyramid.py @@ -4,8 +4,8 @@ import sys import weakref -from pyramid.httpexceptions import HTTPException # type: ignore -from pyramid.request import Request # type: ignore +from pyramid.httpexceptions import HTTPException +from pyramid.request import Request from sentry_sdk.hub import Hub, _should_send_default_pii from sentry_sdk.utils import capture_internal_exceptions, event_from_exception @@ -18,7 +18,7 @@ from sentry_sdk._types import MYPY if MYPY: - from pyramid.response import Response # type: ignore + from pyramid.response import Response from typing import Any from sentry_sdk.integrations.wsgi import _ScopedResponse from typing import Callable @@ -28,6 +28,7 @@ from webob.compat import cgi_FieldStorage # type: ignore from sentry_sdk.utils import ExcInfo + from sentry_sdk._types import EventProcessor if getattr(Request, "authenticated_userid", None): @@ -60,8 +61,8 @@ def __init__(self, transaction_style="route_name"): @staticmethod def setup_once(): # type: () -> None - from pyramid.router import Router # type: ignore - from pyramid.request import Request # type: ignore + from pyramid.router import Router + from pyramid.request import Request old_handle_request = Router.handle_request @@ -83,6 +84,7 @@ def sentry_patched_handle_request(self, request, *args, **kwargs): old_invoke_exception_view = Request.invoke_exception_view def sentry_patched_invoke_exception_view(self, *args, **kwargs): + # type: (Request, *Any, **Any) -> Any rv = old_invoke_exception_view(self, *args, **kwargs) if ( @@ -100,13 +102,14 @@ def sentry_patched_invoke_exception_view(self, *args, **kwargs): old_wsgi_call = Router.__call__ def sentry_patched_wsgi_call(self, environ, start_response): - # type: (Any, Dict[str, str], Callable) -> _ScopedResponse + # type: (Any, Dict[str, str], Callable[..., Any]) -> _ScopedResponse hub = Hub.current integration = hub.get_integration(PyramidIntegration) if integration is None: return old_wsgi_call(self, environ, start_response) def sentry_patched_inner_wsgi_call(environ, start_response): + # type: (Dict[str, Any], Callable[..., Any]) -> Any try: return old_wsgi_call(self, environ, start_response) except Exception: @@ -143,6 +146,7 @@ def _capture_exception(exc_info): class PyramidRequestExtractor(RequestExtractor): def url(self): + # type: () -> str return self.request.path_url def env(self): @@ -183,7 +187,7 @@ def size_of_file(self, postdata): def _make_event_processor(weak_request, integration): - # type: (Callable[[], Request], PyramidIntegration) -> Callable + # type: (Callable[[], Request], PyramidIntegration) -> EventProcessor def event_processor(event, hint): # type: (Dict[str, Any], Dict[str, Any]) -> Dict[str, Any] request = weak_request() diff --git a/sentry_sdk/integrations/redis.py b/sentry_sdk/integrations/redis.py old mode 100755 new mode 100644 index 5e10d3b..510fdbb --- a/sentry_sdk/integrations/redis.py +++ b/sentry_sdk/integrations/redis.py @@ -4,40 +4,67 @@ from sentry_sdk.utils import capture_internal_exceptions from sentry_sdk.integrations import Integration +from sentry_sdk._types import MYPY + +if MYPY: + from typing import Any + class RedisIntegration(Integration): identifier = "redis" @staticmethod def setup_once(): + # type: () -> None import redis - old_execute_command = redis.StrictRedis.execute_command + patch_redis_client(redis.StrictRedis) + + try: + import rb.clients # type: ignore + except ImportError: + pass + else: + patch_redis_client(rb.clients.FanoutClient) + patch_redis_client(rb.clients.MappingClient) + patch_redis_client(rb.clients.RoutingClient) + + +def patch_redis_client(cls): + # type: (Any) -> None + """ + This function can be used to instrument custom redis client classes or + subclasses. + """ + + old_execute_command = cls.execute_command + + def sentry_patched_execute_command(self, name, *args, **kwargs): + # type: (Any, str, *Any, **Any) -> Any + hub = Hub.current - def sentry_patched_execute_command(self, name, *args, **kwargs): - hub = Hub.current + if hub.get_integration(RedisIntegration) is None: + return old_execute_command(self, name, *args, **kwargs) - if hub.get_integration(RedisIntegration) is None: - return old_execute_command(self, name, *args, **kwargs) + description = name - description = name + with capture_internal_exceptions(): + description_parts = [name] + for i, arg in enumerate(args): + if i > 10: + break - with capture_internal_exceptions(): - description_parts = [name] - for i, arg in enumerate(args): - if i > 10: - break + description_parts.append(repr(arg)) - description_parts.append(repr(arg)) + description = " ".join(description_parts) - description = " ".join(description_parts) + with hub.start_span(op="redis", description=description) as span: + if name: + span.set_tag("redis.command", name) - with hub.span(op="redis", description=description) as span: - if name and args and name.lower() in ("get", "set", "setex", "setnx"): - span.set_tag("redis.key", args[0]) + if name and args and name.lower() in ("get", "set", "setex", "setnx"): + span.set_tag("redis.key", args[0]) - return old_execute_command(self, name, *args, **kwargs) + return old_execute_command(self, name, *args, **kwargs) - redis.StrictRedis.execute_command = ( # type: ignore - sentry_patched_execute_command - ) + cls.execute_command = sentry_patched_execute_command diff --git a/sentry_sdk/integrations/rq.py b/sentry_sdk/integrations/rq.py old mode 100755 new mode 100644 index fdc48af..f34afeb --- a/sentry_sdk/integrations/rq.py +++ b/sentry_sdk/integrations/rq.py @@ -7,9 +7,9 @@ from sentry_sdk.tracing import Span from sentry_sdk.utils import capture_internal_exceptions, event_from_exception -from rq.timeouts import JobTimeoutException # type: ignore -from rq.worker import Worker # type: ignore -from rq.queue import Queue # type: ignore +from rq.timeouts import JobTimeoutException +from rq.worker import Worker +from rq.queue import Queue from sentry_sdk._types import MYPY @@ -18,9 +18,10 @@ from typing import Dict from typing import Callable - from rq.job import Job # type: ignore + from rq.job import Job from sentry_sdk.utils import ExcInfo + from sentry_sdk._types import EventProcessor class RqIntegration(Integration): @@ -50,11 +51,12 @@ def sentry_patched_perform_job(self, job, *args, **kwargs): span = Span.continue_from_headers( job.meta.get("_sentry_trace_headers") or {} ) + span.op = "rq.task" with capture_internal_exceptions(): span.transaction = job.func_name - with hub.span(span): + with hub.start_span(span): rv = old_perform_job(self, job, *args, **kwargs) if self.is_horse: @@ -70,6 +72,7 @@ def sentry_patched_perform_job(self, job, *args, **kwargs): old_handle_exception = Worker.handle_exception def sentry_patched_handle_exception(self, job, *exc_info, **kwargs): + # type: (Worker, Any, *Any, **Any) -> Any _capture_exception(exc_info) # type: ignore return old_handle_exception(self, job, *exc_info, **kwargs) @@ -78,6 +81,7 @@ def sentry_patched_handle_exception(self, job, *exc_info, **kwargs): old_enqueue_job = Queue.enqueue_job def sentry_patched_enqueue_job(self, job, **kwargs): + # type: (Queue, Any, **Any) -> Any hub = Hub.current if hub.get_integration(RqIntegration) is not None: job.meta["_sentry_trace_headers"] = dict( @@ -90,7 +94,7 @@ def sentry_patched_enqueue_job(self, job, **kwargs): def _make_event_processor(weak_job): - # type: (Callable[[], Job]) -> Callable + # type: (Callable[[], Job]) -> EventProcessor def event_processor(event, hint): # type: (Dict[str, Any], Dict[str, Any]) -> Dict[str, Any] job = weak_job() diff --git a/sentry_sdk/integrations/sanic.py b/sentry_sdk/integrations/sanic.py old mode 100755 new mode 100644 index 62e8cd2..3016854 --- a/sentry_sdk/integrations/sanic.py +++ b/sentry_sdk/integrations/sanic.py @@ -13,10 +13,10 @@ from sentry_sdk.integrations._wsgi_common import RequestExtractor, _filter_headers from sentry_sdk.integrations.logging import ignore_logger -from sanic import Sanic, __version__ as VERSION # type: ignore -from sanic.exceptions import SanicException # type: ignore -from sanic.router import Router # type: ignore -from sanic.handlers import ErrorHandler # type: ignore +from sanic import Sanic, __version__ as VERSION +from sanic.exceptions import SanicException +from sanic.router import Router +from sanic.handlers import ErrorHandler from sentry_sdk._types import MYPY @@ -26,8 +26,9 @@ from typing import Optional from typing import Union from typing import Tuple + from typing import Dict - from sanic.request import Request, RequestParameters # type: ignore + from sanic.request import Request, RequestParameters from sentry_sdk._types import Event, EventProcessor, Hint @@ -98,7 +99,7 @@ def sentry_router_get(self, request): old_error_handler_lookup = ErrorHandler.lookup def sentry_error_handler_lookup(self, exception): - # type: (Any, Exception) -> Optional[Callable] + # type: (Any, Exception) -> Optional[object] _capture_exception(exception) old_error_handler = old_error_handler_lookup(self, exception) @@ -193,6 +194,7 @@ def content_length(self): return len(self.request.body) def cookies(self): + # type: () -> Dict[str, str] return dict(self.request.cookies) def raw_data(self): @@ -204,6 +206,7 @@ def form(self): return self.request.form def is_json(self): + # type: () -> bool raise NotImplementedError() def json(self): @@ -215,4 +218,5 @@ def files(self): return self.request.files def size_of_file(self, file): + # type: (Any) -> int return len(file.body or ()) diff --git a/sentry_sdk/integrations/serverless.py b/sentry_sdk/integrations/serverless.py old mode 100755 new mode 100644 index 0e20d73..6dd90b4 --- a/sentry_sdk/integrations/serverless.py +++ b/sentry_sdk/integrations/serverless.py @@ -6,10 +6,45 @@ from sentry_sdk._compat import reraise +from sentry_sdk._types import MYPY + +if MYPY: + from typing import Any + from typing import Callable + from typing import TypeVar + from typing import Union + from typing import Optional + + from typing import overload + + F = TypeVar("F", bound=Callable[..., Any]) + +else: + + def overload(x): + # type: (F) -> F + return x + + +@overload +def serverless_function(f, flush=True): + # type: (F, bool) -> F + pass + + +@overload # noqa def serverless_function(f=None, flush=True): + # type: (None, bool) -> Callable[[F], F] + pass + + +def serverless_function(f=None, flush=True): # noqa + # type: (Optional[F], bool) -> Union[F, Callable[[F], F]] def wrapper(f): + # type: (F) -> F @functools.wraps(f) def inner(*args, **kwargs): + # type: (*Any, **Any) -> Any with Hub(Hub.current) as hub: with hub.configure_scope() as scope: scope.clear_breadcrumbs() @@ -22,7 +57,7 @@ def inner(*args, **kwargs): if flush: _flush_client() - return inner + return inner # type: ignore if f is None: return wrapper @@ -31,6 +66,7 @@ def inner(*args, **kwargs): def _capture_and_reraise(): + # type: () -> None exc_info = sys.exc_info() hub = Hub.current if hub is not None and hub.client is not None: @@ -45,6 +81,7 @@ def _capture_and_reraise(): def _flush_client(): + # type: () -> None hub = Hub.current if hub is not None: hub.flush() diff --git a/sentry_sdk/integrations/spark/__init__.py b/sentry_sdk/integrations/spark/__init__.py new file mode 100644 index 0000000..10d9416 --- /dev/null +++ b/sentry_sdk/integrations/spark/__init__.py @@ -0,0 +1,4 @@ +from sentry_sdk.integrations.spark.spark_driver import SparkIntegration +from sentry_sdk.integrations.spark.spark_worker import SparkWorkerIntegration + +__all__ = ["SparkIntegration", "SparkWorkerIntegration"] diff --git a/sentry_sdk/integrations/spark/spark_driver.py b/sentry_sdk/integrations/spark/spark_driver.py new file mode 100644 index 0000000..1c4fde1 --- /dev/null +++ b/sentry_sdk/integrations/spark/spark_driver.py @@ -0,0 +1,261 @@ +from sentry_sdk import configure_scope +from sentry_sdk.hub import Hub +from sentry_sdk.integrations import Integration +from sentry_sdk.utils import capture_internal_exceptions + +from sentry_sdk._types import MYPY + +if MYPY: + from typing import Any + from typing import Optional + + from sentry_sdk._types import Event, Hint + + +class SparkIntegration(Integration): + identifier = "spark" + + @staticmethod + def setup_once(): + # type: () -> None + patch_spark_context_init() + + +def _set_app_properties(): + # type: () -> None + """ + Set properties in driver that propagate to worker processes, allowing for workers to have access to those properties. + This allows worker integration to have access to app_name and application_id. + """ + from pyspark import SparkContext + + sparkContext = SparkContext._active_spark_context + if sparkContext: + sparkContext.setLocalProperty("sentry_app_name", sparkContext.appName) + sparkContext.setLocalProperty( + "sentry_application_id", sparkContext.applicationId + ) + + +def _start_sentry_listener(sc): + # type: (Any) -> None + """ + Start java gateway server to add custom `SparkListener` + """ + from pyspark.java_gateway import ensure_callback_server_started + + gw = sc._gateway + ensure_callback_server_started(gw) + listener = SentryListener() + sc._jsc.sc().addSparkListener(listener) + + +def patch_spark_context_init(): + # type: () -> None + from pyspark import SparkContext + + spark_context_init = SparkContext._do_init + + def _sentry_patched_spark_context_init(self, *args, **kwargs): + # type: (SparkContext, *Any, **Any) -> Optional[Any] + init = spark_context_init(self, *args, **kwargs) + + if Hub.current.get_integration(SparkIntegration) is None: + return init + + _start_sentry_listener(self) + _set_app_properties() + + with configure_scope() as scope: + + @scope.add_event_processor + def process_event(event, hint): + # type: (Event, Hint) -> Optional[Event] + with capture_internal_exceptions(): + if Hub.current.get_integration(SparkIntegration) is None: + return event + + event.setdefault("user", {}).setdefault("id", self.sparkUser()) + + event.setdefault("tags", {}).setdefault( + "executor.id", self._conf.get("spark.executor.id") + ) + event["tags"].setdefault( + "spark-submit.deployMode", + self._conf.get("spark.submit.deployMode"), + ) + event["tags"].setdefault( + "driver.host", self._conf.get("spark.driver.host") + ) + event["tags"].setdefault( + "driver.port", self._conf.get("spark.driver.port") + ) + event["tags"].setdefault("spark_version", self.version) + event["tags"].setdefault("app_name", self.appName) + event["tags"].setdefault("application_id", self.applicationId) + event["tags"].setdefault("master", self.master) + event["tags"].setdefault("spark_home", self.sparkHome) + + event.setdefault("extra", {}).setdefault("web_url", self.uiWebUrl) + + return event + + return init + + SparkContext._do_init = _sentry_patched_spark_context_init + + +class SparkListener(object): + def onApplicationEnd(self, applicationEnd): + # type: (Any) -> None + pass + + def onApplicationStart(self, applicationStart): + # type: (Any) -> None + pass + + def onBlockManagerAdded(self, blockManagerAdded): + # type: (Any) -> None + pass + + def onBlockManagerRemoved(self, blockManagerRemoved): + # type: (Any) -> None + pass + + def onBlockUpdated(self, blockUpdated): + # type: (Any) -> None + pass + + def onEnvironmentUpdate(self, environmentUpdate): + # type: (Any) -> None + pass + + def onExecutorAdded(self, executorAdded): + # type: (Any) -> None + pass + + def onExecutorBlacklisted(self, executorBlacklisted): + # type: (Any) -> None + pass + + def onExecutorBlacklistedForStage(self, executorBlacklistedForStage): + # type: (Any) -> None + pass + + def onExecutorMetricsUpdate(self, executorMetricsUpdate): + # type: (Any) -> None + pass + + def onExecutorRemoved(self, executorRemoved): + # type: (Any) -> None + pass + + def onJobEnd(self, jobEnd): + # type: (Any) -> None + pass + + def onJobStart(self, jobStart): + # type: (Any) -> None + pass + + def onNodeBlacklisted(self, nodeBlacklisted): + # type: (Any) -> None + pass + + def onNodeBlacklistedForStage(self, nodeBlacklistedForStage): + # type: (Any) -> None + pass + + def onNodeUnblacklisted(self, nodeUnblacklisted): + # type: (Any) -> None + pass + + def onOtherEvent(self, event): + # type: (Any) -> None + pass + + def onSpeculativeTaskSubmitted(self, speculativeTask): + # type: (Any) -> None + pass + + def onStageCompleted(self, stageCompleted): + # type: (Any) -> None + pass + + def onStageSubmitted(self, stageSubmitted): + # type: (Any) -> None + pass + + def onTaskEnd(self, taskEnd): + # type: (Any) -> None + pass + + def onTaskGettingResult(self, taskGettingResult): + # type: (Any) -> None + pass + + def onTaskStart(self, taskStart): + # type: (Any) -> None + pass + + def onUnpersistRDD(self, unpersistRDD): + # type: (Any) -> None + pass + + class Java: + implements = ["org.apache.spark.scheduler.SparkListenerInterface"] + + +class SentryListener(SparkListener): + def __init__(self): + # type: () -> None + self.hub = Hub.current + + def onJobStart(self, jobStart): + # type: (Any) -> None + message = "Job {} Started".format(jobStart.jobId()) + self.hub.add_breadcrumb(level="info", message=message) + _set_app_properties() + + def onJobEnd(self, jobEnd): + # type: (Any) -> None + level = "" + message = "" + data = {"result": jobEnd.jobResult().toString()} + + if jobEnd.jobResult().toString() == "JobSucceeded": + level = "info" + message = "Job {} Ended".format(jobEnd.jobId()) + else: + level = "warning" + message = "Job {} Failed".format(jobEnd.jobId()) + + self.hub.add_breadcrumb(level=level, message=message, data=data) + + def onStageSubmitted(self, stageSubmitted): + # type: (Any) -> None + stageInfo = stageSubmitted.stageInfo() + message = "Stage {} Submitted".format(stageInfo.stageId()) + data = {"attemptId": stageInfo.attemptId(), "name": stageInfo.name()} + self.hub.add_breadcrumb(level="info", message=message, data=data) + _set_app_properties() + + def onStageCompleted(self, stageCompleted): + # type: (Any) -> None + from py4j.protocol import Py4JJavaError # type: ignore + + stageInfo = stageCompleted.stageInfo() + message = "" + level = "" + data = {"attemptId": stageInfo.attemptId(), "name": stageInfo.name()} + + # Have to Try Except because stageInfo.failureReason() is typed with Scala Option + try: + data["reason"] = stageInfo.failureReason().get() + message = "Stage {} Failed".format(stageInfo.stageId()) + level = "warning" + except Py4JJavaError: + message = "Stage {} Completed".format(stageInfo.stageId()) + level = "info" + + self.hub.add_breadcrumb(level=level, message=message, data=data) diff --git a/sentry_sdk/integrations/spark/spark_worker.py b/sentry_sdk/integrations/spark/spark_worker.py new file mode 100644 index 0000000..4d0b7fa --- /dev/null +++ b/sentry_sdk/integrations/spark/spark_worker.py @@ -0,0 +1,120 @@ +from __future__ import absolute_import + +import sys + +from sentry_sdk import configure_scope +from sentry_sdk.hub import Hub +from sentry_sdk.integrations import Integration +from sentry_sdk.utils import ( + capture_internal_exceptions, + exc_info_from_error, + single_exception_from_error_tuple, + walk_exception_chain, + event_hint_with_exc_info, +) + +from sentry_sdk._types import MYPY + +if MYPY: + from typing import Any + from typing import Optional + + from sentry_sdk._types import ExcInfo, Event, Hint + + +class SparkWorkerIntegration(Integration): + identifier = "spark_worker" + + @staticmethod + def setup_once(): + # type: () -> None + import pyspark.daemon as original_daemon + + original_daemon.worker_main = _sentry_worker_main + + +def _capture_exception(exc_info, hub): + # type: (ExcInfo, Hub) -> None + client = hub.client + + client_options = client.options # type: ignore + + mechanism = {"type": "spark", "handled": False} + + exc_info = exc_info_from_error(exc_info) + + exc_type, exc_value, tb = exc_info + rv = [] + + # On Exception worker will call sys.exit(-1), so we can ignore SystemExit and similar errors + for exc_type, exc_value, tb in walk_exception_chain(exc_info): + if exc_type not in (SystemExit, EOFError, ConnectionResetError): + rv.append( + single_exception_from_error_tuple( + exc_type, exc_value, tb, client_options, mechanism + ) + ) + + if rv: + rv.reverse() + hint = event_hint_with_exc_info(exc_info) + event = {"level": "error", "exception": {"values": rv}} + + _tag_task_context() + + hub.capture_event(event, hint=hint) + + +def _tag_task_context(): + # type: () -> None + from pyspark.taskcontext import TaskContext + + with configure_scope() as scope: + + @scope.add_event_processor + def process_event(event, hint): + # type: (Event, Hint) -> Optional[Event] + with capture_internal_exceptions(): + integration = Hub.current.get_integration(SparkWorkerIntegration) + taskContext = TaskContext.get() + + if integration is None or taskContext is None: + return event + + event.setdefault("tags", {}).setdefault( + "stageId", taskContext.stageId() + ) + event["tags"].setdefault("partitionId", taskContext.partitionId()) + event["tags"].setdefault("attemptNumber", taskContext.attemptNumber()) + event["tags"].setdefault("taskAttemptId", taskContext.taskAttemptId()) + + if taskContext._localProperties: + if "sentry_app_name" in taskContext._localProperties: + event["tags"].setdefault( + "app_name", taskContext._localProperties["sentry_app_name"] + ) + event["tags"].setdefault( + "application_id", + taskContext._localProperties["sentry_application_id"], + ) + + if "callSite.short" in taskContext._localProperties: + event.setdefault("extra", {}).setdefault( + "callSite", taskContext._localProperties["callSite.short"] + ) + + return event + + +def _sentry_worker_main(*args, **kwargs): + # type: (*Optional[Any], **Optional[Any]) -> None + import pyspark.worker as original_worker + + try: + original_worker.main(*args, **kwargs) + except SystemExit: + if Hub.current.get_integration(SparkWorkerIntegration) is not None: + hub = Hub.current + exc_info = sys.exc_info() + with capture_internal_exceptions(): + _capture_exception(exc_info, hub) diff --git a/sentry_sdk/integrations/sqlalchemy.py b/sentry_sdk/integrations/sqlalchemy.py new file mode 100644 index 0000000..a5f2a0d --- /dev/null +++ b/sentry_sdk/integrations/sqlalchemy.py @@ -0,0 +1,71 @@ +from __future__ import absolute_import + +from sentry_sdk._types import MYPY +from sentry_sdk.hub import Hub +from sentry_sdk.integrations import Integration +from sentry_sdk.tracing import record_sql_queries + +from sqlalchemy.engine import Engine # type: ignore +from sqlalchemy.event import listen # type: ignore + +if MYPY: + from typing import Any + from typing import ContextManager + from typing import Optional + + from sentry_sdk.tracing import Span + + +class SqlalchemyIntegration(Integration): + identifier = "sqlalchemy" + + @staticmethod + def setup_once(): + # type: () -> None + + listen(Engine, "before_cursor_execute", _before_cursor_execute) + listen(Engine, "after_cursor_execute", _after_cursor_execute) + listen(Engine, "dbapi_error", _dbapi_error) + + +def _before_cursor_execute( + conn, cursor, statement, parameters, context, executemany, *args +): + # type: (Any, Any, Any, Any, Any, bool, *Any) -> None + hub = Hub.current + if hub.get_integration(SqlalchemyIntegration) is None: + return + + ctx_mgr = record_sql_queries( + hub, + cursor, + statement, + parameters, + paramstyle=context and context.dialect and context.dialect.paramstyle or None, + executemany=executemany, + ) + conn._sentry_sql_span_manager = ctx_mgr + + span = ctx_mgr.__enter__() + + if span is not None: + conn._sentry_sql_span = span + + +def _after_cursor_execute(conn, cursor, statement, *args): + # type: (Any, Any, Any, *Any) -> None + ctx_mgr = getattr( + conn, "_sentry_sql_span_manager", None + ) # type: ContextManager[Any] + + if ctx_mgr is not None: + conn._sentry_sql_span_manager = None + ctx_mgr.__exit__(None, None, None) + + +def _dbapi_error(conn, *args): + # type: (Any, *Any) -> None + span = getattr(conn, "_sentry_sql_span", None) # type: Optional[Span] + + if span is not None: + span.set_status("internal_error") diff --git a/sentry_sdk/integrations/stdlib.py b/sentry_sdk/integrations/stdlib.py old mode 100755 new mode 100644 index 66ab126..56cece7 --- a/sentry_sdk/integrations/stdlib.py +++ b/sentry_sdk/integrations/stdlib.py @@ -6,13 +6,27 @@ from sentry_sdk.hub import Hub from sentry_sdk.integrations import Integration from sentry_sdk.scope import add_global_event_processor -from sentry_sdk.tracing import EnvironHeaders, record_http_request +from sentry_sdk.tracing import EnvironHeaders +from sentry_sdk.utils import capture_internal_exceptions, safe_repr + +from sentry_sdk._types import MYPY + +if MYPY: + from typing import Any + from typing import Callable + from typing import Dict + from typing import Optional + from typing import List + + from sentry_sdk._types import Event, Hint + try: from httplib import HTTPConnection # type: ignore except ImportError: from http.client import HTTPConnection + _RUNTIME_CONTEXT = { "name": platform.python_implementation(), "version": "%s.%s.%s" % (sys.version_info[:3]), @@ -31,6 +45,7 @@ def setup_once(): @add_global_event_processor def add_python_runtime_context(event, hint): + # type: (Event, Hint) -> Optional[Event] if Hub.current.get_integration(StdlibIntegration) is not None: contexts = event.setdefault("contexts", {}) if isinstance(contexts, dict) and "runtime" not in contexts: @@ -45,6 +60,7 @@ def _install_httplib(): real_getresponse = HTTPConnection.getresponse def putrequest(self, method, url, *args, **kwargs): + # type: (HTTPConnection, str, str, *Any, **Any) -> Any hub = Hub.current if hub.get_integration(StdlibIntegration) is None: return real_putrequest(self, method, url, *args, **kwargs) @@ -62,48 +78,33 @@ def putrequest(self, method, url, *args, **kwargs): url, ) - recorder = record_http_request(hub, real_url, method) - data_dict = recorder.__enter__() + span = hub.start_span(op="http", description="%s %s" % (method, real_url)) + + span.set_data("method", method) + span.set_data("url", real_url) - try: - rv = real_putrequest(self, method, url, *args, **kwargs) + rv = real_putrequest(self, method, url, *args, **kwargs) - for key, value in hub.iter_trace_propagation_headers(): - self.putheader(key, value) - except Exception: - recorder.__exit__(*sys.exc_info()) - raise + for key, value in hub.iter_trace_propagation_headers(): + self.putheader(key, value) - self._sentrysdk_recorder = recorder - self._sentrysdk_data_dict = data_dict + self._sentrysdk_span = span return rv def getresponse(self, *args, **kwargs): - recorder = getattr(self, "_sentrysdk_recorder", None) + # type: (HTTPConnection, *Any, **Any) -> Any + span = getattr(self, "_sentrysdk_span", None) - if recorder is None: + if span is None: return real_getresponse(self, *args, **kwargs) - data_dict = getattr(self, "_sentrysdk_data_dict", None) - - try: - rv = real_getresponse(self, *args, **kwargs) - - if data_dict is not None: - data_dict["httplib_response"] = rv - data_dict["status_code"] = rv.status - data_dict["reason"] = rv.reason - except TypeError: - # python-requests provokes a typeerror to discover py3 vs py2 differences - # - # > TypeError("getresponse() got an unexpected keyword argument 'buffering'") - raise - except Exception: - recorder.__exit__(*sys.exc_info()) - raise - else: - recorder.__exit__(None, None, None) + rv = real_getresponse(self, *args, **kwargs) + + span.set_data("status_code", rv.status) + span.set_http_status(int(rv.status)) + span.set_data("reason", rv.reason) + span.finish() return rv @@ -111,45 +112,119 @@ def getresponse(self, *args, **kwargs): HTTPConnection.getresponse = getresponse -def _get_argument(args, kwargs, name, position, setdefault=None): +def _init_argument(args, kwargs, name, position, setdefault_callback=None): + # type: (List[Any], Dict[Any, Any], str, int, Optional[Callable[[Any], Any]]) -> Any + """ + given (*args, **kwargs) of a function call, retrieve (and optionally set a + default for) an argument by either name or position. + + This is useful for wrapping functions with complex type signatures and + extracting a few arguments without needing to redefine that function's + entire type signature. + """ + if name in kwargs: rv = kwargs[name] - if rv is None and setdefault is not None: - rv = kwargs[name] = setdefault + if setdefault_callback is not None: + rv = setdefault_callback(rv) + if rv is not None: + kwargs[name] = rv elif position < len(args): rv = args[position] - if rv is None and setdefault is not None: - rv = args[position] = setdefault + if setdefault_callback is not None: + rv = setdefault_callback(rv) + if rv is not None: + args[position] = rv else: - rv = kwargs[name] = setdefault + rv = setdefault_callback and setdefault_callback(None) + if rv is not None: + kwargs[name] = rv return rv def _install_subprocess(): + # type: () -> None old_popen_init = subprocess.Popen.__init__ def sentry_patched_popen_init(self, *a, **kw): + # type: (subprocess.Popen[Any], *Any, **Any) -> None + hub = Hub.current if hub.get_integration(StdlibIntegration) is None: - return old_popen_init(self, *a, **kw) + return old_popen_init(self, *a, **kw) # type: ignore + + # Convert from tuple to list to be able to set values. + a = list(a) + + args = _init_argument(a, kw, "args", 0) or [] + cwd = _init_argument(a, kw, "cwd", 9) + + # if args is not a list or tuple (and e.g. some iterator instead), + # let's not use it at all. There are too many things that can go wrong + # when trying to collect an iterator into a list and setting that list + # into `a` again. + # + # Also invocations where `args` is not a sequence are not actually + # legal. They just happen to work under CPython. + description = None + + if isinstance(args, (list, tuple)) and len(args) < 100: + with capture_internal_exceptions(): + description = " ".join(map(str, args)) + + if description is None: + description = safe_repr(args) - # do not setdefault! args is required by Popen, doing setdefault would - # make invalid calls valid - args = _get_argument(a, kw, "args", 0) or [] - cwd = _get_argument(a, kw, "cwd", 10) + env = None for k, v in hub.iter_trace_propagation_headers(): - env = _get_argument(a, kw, "env", 11, {}) + if env is None: + env = _init_argument(a, kw, "env", 10, lambda x: dict(x or os.environ)) env["SUBPROCESS_" + k.upper().replace("-", "_")] = v - with hub.span(op="subprocess", description=" ".join(map(str, args))) as span: - span.set_tag("subprocess.cwd", cwd) + with hub.start_span(op="subprocess", description=description) as span: + if cwd: + span.set_data("subprocess.cwd", cwd) - return old_popen_init(self, *a, **kw) + rv = old_popen_init(self, *a, **kw) # type: ignore + + span.set_tag("subprocess.pid", self.pid) + return rv subprocess.Popen.__init__ = sentry_patched_popen_init # type: ignore + old_popen_wait = subprocess.Popen.wait + + def sentry_patched_popen_wait(self, *a, **kw): + # type: (subprocess.Popen[Any], *Any, **Any) -> Any + hub = Hub.current + + if hub.get_integration(StdlibIntegration) is None: + return old_popen_wait(self, *a, **kw) + + with hub.start_span(op="subprocess.wait") as span: + span.set_tag("subprocess.pid", self.pid) + return old_popen_wait(self, *a, **kw) + + subprocess.Popen.wait = sentry_patched_popen_wait # type: ignore + + old_popen_communicate = subprocess.Popen.communicate + + def sentry_patched_popen_communicate(self, *a, **kw): + # type: (subprocess.Popen[Any], *Any, **Any) -> Any + hub = Hub.current + + if hub.get_integration(StdlibIntegration) is None: + return old_popen_communicate(self, *a, **kw) + + with hub.start_span(op="subprocess.communicate") as span: + span.set_tag("subprocess.pid", self.pid) + return old_popen_communicate(self, *a, **kw) + + subprocess.Popen.communicate = sentry_patched_popen_communicate # type: ignore + def get_subprocess_traceparent_headers(): + # type: () -> EnvironHeaders return EnvironHeaders(os.environ, prefix="SUBPROCESS_") diff --git a/sentry_sdk/integrations/threading.py b/sentry_sdk/integrations/threading.py old mode 100755 new mode 100644 index 3bd6032..b750257 --- a/sentry_sdk/integrations/threading.py +++ b/sentry_sdk/integrations/threading.py @@ -1,24 +1,30 @@ from __future__ import absolute_import import sys - -from threading import Thread +from threading import Thread, current_thread from sentry_sdk import Hub from sentry_sdk._compat import reraise -from sentry_sdk.utils import event_from_exception -from sentry_sdk.integrations import Integration - from sentry_sdk._types import MYPY +from sentry_sdk.integrations import Integration +from sentry_sdk.utils import event_from_exception, capture_internal_exceptions if MYPY: from typing import Any + from typing import TypeVar + from typing import Callable + from typing import Optional + + from sentry_sdk._types import ExcInfo + + F = TypeVar("F", bound=Callable[..., Any]) class ThreadingIntegration(Integration): identifier = "threading" def __init__(self, propagate_hub=False): + # type: (bool) -> None self.propagate_hub = propagate_hub @staticmethod @@ -27,6 +33,7 @@ def setup_once(): old_start = Thread.start def sentry_start(self, *a, **kw): + # type: (Thread, *Any, **Any) -> Any hub = Hub.current integration = hub.get_integration(ThreadingIntegration) if integration is not None: @@ -34,28 +41,38 @@ def sentry_start(self, *a, **kw): hub_ = None else: hub_ = Hub(hub) - - self.run = _wrap_run(hub_, self.run) + # Patching instance methods in `start()` creates a reference cycle if + # done in a naive way. See + # https://github.com/getsentry/sentry-python/pull/434 + # + # In threading module, using current_thread API will access current thread instance + # without holding it to avoid a reference cycle in an easier way. + with capture_internal_exceptions(): + new_run = _wrap_run(hub_, getattr(self.run, "__func__", self.run)) + self.run = new_run # type: ignore return old_start(self, *a, **kw) # type: ignore Thread.start = sentry_start # type: ignore -def _wrap_run(parent_hub, old_run): +def _wrap_run(parent_hub, old_run_func): + # type: (Optional[Hub], F) -> F def run(*a, **kw): + # type: (*Any, **Any) -> Any hub = parent_hub or Hub.current - with hub: try: - return old_run(*a, **kw) + self = current_thread() + return old_run_func(self, *a, **kw) except Exception: reraise(*_capture_exception()) - return run + return run # type: ignore def _capture_exception(): + # type: () -> ExcInfo hub = Hub.current exc_info = sys.exc_info() diff --git a/sentry_sdk/integrations/tornado.py b/sentry_sdk/integrations/tornado.py old mode 100755 new mode 100644 index eaa6806..3c43e01 --- a/sentry_sdk/integrations/tornado.py +++ b/sentry_sdk/integrations/tornado.py @@ -17,18 +17,19 @@ from sentry_sdk.integrations.logging import ignore_logger from sentry_sdk._compat import iteritems -from tornado.web import RequestHandler, HTTPError # type: ignore -from tornado.gen import coroutine # type: ignore +from tornado.web import RequestHandler, HTTPError +from tornado.gen import coroutine from sentry_sdk._types import MYPY if MYPY: from typing import Any - from typing import List from typing import Optional from typing import Dict from typing import Callable + from sentry_sdk._types import EventProcessor + class TornadoIntegration(Integration): identifier = "tornado" @@ -36,7 +37,7 @@ class TornadoIntegration(Integration): @staticmethod def setup_once(): # type: () -> None - import tornado # type: ignore + import tornado tornado_version = getattr(tornado, "version_info", None) if tornado_version is None or tornado_version < (5, 0): @@ -49,7 +50,6 @@ def setup_once(): "The tornado integration for Sentry requires Python 3.6+ or the aiocontextvars package" ) - ignore_logger("tornado.application") ignore_logger("tornado.access") old_execute = RequestHandler._execute @@ -60,7 +60,7 @@ def setup_once(): # Starting Tornado 6 RequestHandler._execute method is a standard Python coroutine (async/await) # In that case our method should be a coroutine function too async def sentry_execute_request_handler(self, *args, **kwargs): - # type: (Any, *List, **Any) -> Any + # type: (Any, *Any, **Any) -> Any hub = Hub.current integration = hub.get_integration(TornadoIntegration) if integration is None: @@ -78,6 +78,7 @@ async def sentry_execute_request_handler(self, *args, **kwargs): @coroutine # type: ignore def sentry_execute_request_handler(self, *args, **kwargs): + # type: (RequestHandler, *Any, **Any) -> Any hub = Hub.current integration = hub.get_integration(TornadoIntegration) if integration is None: @@ -124,7 +125,7 @@ def _capture_exception(ty, value, tb): def _make_event_processor(weak_handler): - # type: (Callable[[], RequestHandler]) -> Callable + # type: (Callable[[], RequestHandler]) -> EventProcessor def tornado_processor(event, hint): # type: (Dict[str, Any], Dict[str, Any]) -> Dict[str, Any] handler = weak_handler() @@ -171,7 +172,7 @@ def content_length(self): return len(self.request.body) def cookies(self): - # type: () -> Dict + # type: () -> Dict[str, str] return {k: v.value for k, v in iteritems(self.request.cookies)} def raw_data(self): @@ -179,7 +180,7 @@ def raw_data(self): return self.request.body def form(self): - # type: () -> Optional[Any] + # type: () -> Dict[str, Any] return { k: [v.decode("latin1", "replace") for v in vs] for k, vs in iteritems(self.request.body_arguments) @@ -190,8 +191,9 @@ def is_json(self): return _is_json_content_type(self.request.headers.get("content-type")) def files(self): - # type: () -> Dict + # type: () -> Dict[str, Any] return {k: v[0] for k, v in iteritems(self.request.files) if v} def size_of_file(self, file): + # type: (Any) -> int return len(file.body or ()) diff --git a/sentry_sdk/integrations/wsgi.py b/sentry_sdk/integrations/wsgi.py old mode 100755 new mode 100644 index ea98fb8..8b881bc --- a/sentry_sdk/integrations/wsgi.py +++ b/sentry_sdk/integrations/wsgi.py @@ -1,7 +1,12 @@ +import functools import sys from sentry_sdk.hub import Hub, _should_send_default_pii -from sentry_sdk.utils import capture_internal_exceptions, event_from_exception +from sentry_sdk.utils import ( + ContextVar, + capture_internal_exceptions, + event_from_exception, +) from sentry_sdk._compat import PY2, reraise, iteritems from sentry_sdk.tracing import Span from sentry_sdk.integrations._wsgi_common import _filter_headers @@ -11,13 +16,21 @@ if MYPY: from typing import Callable from typing import Dict - from typing import List from typing import Iterator from typing import Any from typing import Tuple from typing import Optional + from typing import TypeVar from sentry_sdk.utils import ExcInfo + from sentry_sdk._types import EventProcessor + + T = TypeVar("T") + U = TypeVar("U") + E = TypeVar("E") + + +_wsgi_middleware_applied = ContextVar("sentry_wsgi_middleware_applied") if PY2: @@ -72,30 +85,54 @@ class SentryWsgiMiddleware(object): __slots__ = ("app",) def __init__(self, app): - # type: (Callable) -> None + # type: (Callable[[Dict[str, str], Callable[..., Any]], Any]) -> None self.app = app def __call__(self, environ, start_response): - # type: (Dict[str, str], Callable) -> _ScopedResponse - hub = Hub(Hub.current) + # type: (Dict[str, str], Callable[..., Any]) -> _ScopedResponse + if _wsgi_middleware_applied.get(False): + return self.app(environ, start_response) + + _wsgi_middleware_applied.set(True) + try: + hub = Hub(Hub.current) + + with hub: + with capture_internal_exceptions(): + with hub.configure_scope() as scope: + scope.clear_breadcrumbs() + scope._name = "wsgi" + scope.add_event_processor(_make_wsgi_event_processor(environ)) + + span = Span.continue_from_environ(environ) + span.op = "http.server" + span.transaction = "generic WSGI request" + + with hub.start_span(span) as span: + try: + rv = self.app( + environ, + functools.partial( + _sentry_start_response, start_response, span + ), + ) + except BaseException: + reraise(*_capture_exception(hub)) + finally: + _wsgi_middleware_applied.set(False) - with hub: - with capture_internal_exceptions(): - with hub.configure_scope() as scope: - scope.clear_breadcrumbs() - scope._name = "wsgi" - scope.add_event_processor(_make_wsgi_event_processor(environ)) + return _ScopedResponse(hub, rv) - span = Span.continue_from_environ(environ) - span.transaction = environ.get("PATH_INFO") or "unknown http request" - with hub.span(span): - try: - rv = self.app(environ, start_response) - except BaseException: - reraise(*_capture_exception(hub)) +def _sentry_start_response( + old_start_response, span, status, response_headers, exc_info=None +): + # type: (Callable[[str, U, Optional[E]], T], Span, str, U, Optional[E]) -> T + with capture_internal_exceptions(): + status_int = int(status.split(" ", 1)[0]) + span.set_http_status(status_int) - return _ScopedResponse(hub, rv) + return old_start_response(status, response_headers, exc_info) def _get_environ(environ): @@ -180,7 +217,7 @@ class _ScopedResponse(object): __slots__ = ("_response", "_hub") def __init__(self, hub, response): - # type: (Hub, List[bytes]) -> None + # type: (Hub, Iterator[bytes]) -> None self._hub = hub self._response = response @@ -200,9 +237,10 @@ def __iter__(self): yield chunk def close(self): + # type: () -> None with self._hub: try: - self._response.close() + self._response.close() # type: ignore except AttributeError: pass except BaseException: @@ -210,7 +248,7 @@ def close(self): def _make_wsgi_event_processor(environ): - # type: (Dict[str, str]) -> Callable + # type: (Dict[str, str]) -> EventProcessor # It's a bit unfortunate that we have to extract and parse the request data # from the environ so eagerly, but there are a few good reasons for this. # @@ -238,7 +276,8 @@ def event_processor(event, hint): if _should_send_default_pii(): user_info = event.setdefault("user", {}) - user_info["ip_address"] = client_ip + if client_ip: + user_info["ip_address"] = client_ip request_info["url"] = request_url request_info["query_string"] = query_string diff --git a/sentry_sdk/py.typed b/sentry_sdk/py.typed old mode 100755 new mode 100644 diff --git a/sentry_sdk/scope.py b/sentry_sdk/scope.py old mode 100755 new mode 100644 index 7265700..1ea2f11 --- a/sentry_sdk/scope.py +++ b/sentry_sdk/scope.py @@ -4,7 +4,6 @@ from itertools import chain from sentry_sdk.utils import logger, capture_internal_exceptions - from sentry_sdk._types import MYPY if MYPY: @@ -21,10 +20,15 @@ Event, EventProcessor, ErrorProcessor, + ExcInfo, Hint, + Type, ) + from sentry_sdk.tracing import Span + F = TypeVar("F", bound=Callable[..., Any]) + T = TypeVar("T") global_event_processors = [] # type: List[EventProcessor] @@ -36,6 +40,7 @@ def add_global_event_processor(processor): def _attr_setter(fn): + # type: (Any) -> Any return property(fset=fn, doc=fn.__doc__) @@ -60,6 +65,12 @@ class Scope(object): events that belong to it. """ + # NOTE: Even though it should not happen, the scope needs to not crash when + # accessed by multiple threads. It's fine if it's full of races, but those + # races should never make the user application crash. + # + # The same needs to hold for any accesses of the scope the SDK makes. + __slots__ = ( "_level", "_name", @@ -84,38 +95,74 @@ def __init__(self): self._name = None # type: Optional[str] self.clear() + def clear(self): + # type: () -> None + """Clears the entire scope.""" + self._level = None # type: Optional[str] + self._fingerprint = None # type: Optional[List[str]] + self._transaction = None # type: Optional[str] + self._user = None # type: Optional[Dict[str, Any]] + + self._tags = {} # type: Dict[str, Any] + self._contexts = {} # type: Dict[str, Dict[str, Any]] + self._extras = {} # type: Dict[str, Any] + + self.clear_breadcrumbs() + self._should_capture = True + + self._span = None # type: Optional[Span] + @_attr_setter def level(self, value): - """When set this overrides the level.""" + # type: (Optional[str]) -> None + """When set this overrides the level. Deprecated in favor of set_level.""" + self._level = value + + def set_level(self, value): + # type: (Optional[str]) -> None + """Sets the level for the scope.""" self._level = value @_attr_setter def fingerprint(self, value): + # type: (Optional[List[str]]) -> None """When set this overrides the default fingerprint.""" self._fingerprint = value @_attr_setter def transaction(self, value): + # type: (Optional[str]) -> None """When set this forces a specific transaction name to be set.""" self._transaction = value - if self._span: - self._span.transaction = value + span = self._span + if span: + span.transaction = value @_attr_setter def user(self, value): - """When set a specific user is bound to the scope.""" + # type: (Dict[str, Any]) -> None + """When set a specific user is bound to the scope. Deprecated in favor of set_user.""" + self._user = value + + def set_user(self, value): + # type: (Dict[str, Any]) -> None + """Sets a user for the scope.""" self._user = value @property def span(self): + # type: () -> Optional[Span] """Get/set current tracing span.""" return self._span @span.setter def span(self, span): + # type: (Optional[Span]) -> None self._span = span - if span is not None and span.transaction: - self._transaction = span.transaction + if span is not None: + span_transaction = span.transaction + if span_transaction: + self._transaction = span_transaction def set_tag( self, @@ -165,23 +212,6 @@ def remove_extra( """Removes a specific extra key.""" self._extras.pop(key, None) - def clear(self): - # type: () -> None - """Clears the entire scope.""" - self._level = None - self._fingerprint = None - self._transaction = None - self._user = None - - self._tags = {} # type: Dict[str, Any] - self._contexts = {} # type: Dict[str, Dict[str, Any]] - self._extras = {} # type: Dict[str, Any] - - self.clear_breadcrumbs() - self._should_capture = True - - self._span = None - def clear_breadcrumbs(self): # type: () -> None """Clears breadcrumb buffer.""" @@ -195,12 +225,19 @@ def add_event_processor( :param func: This function behaves like `before_send.` """ + if len(self._event_processors) > 20: + logger.warning( + "Too many event processors on scope! Clearing list to free up some memory: %r", + self._event_processors, + ) + del self._event_processors[:] + self._event_processors.append(func) def add_error_processor( self, func, # type: ErrorProcessor - cls=None, # type: Optional[type] + cls=None, # type: Optional[Type[BaseException]] ): # type: (...) -> None """Register a scope local error processor on the scope. @@ -214,6 +251,7 @@ def add_error_processor( real_func = func def func(event, exc_info): + # type: (Event, ExcInfo) -> Optional[Event] try: is_inst = isinstance(exc_info[1], cls_) except Exception: @@ -241,7 +279,9 @@ def _drop(event, cause, ty): if self._level is not None: event["level"] = self._level - event.setdefault("breadcrumbs", []).extend(self._breadcrumbs) + if event.get("type") != "transaction": + event.setdefault("breadcrumbs", []).extend(self._breadcrumbs) + if event.get("user") is None and self._user is not None: event["user"] = self._user diff --git a/sentry_sdk/serializer.py b/sentry_sdk/serializer.py old mode 100755 new mode 100644 index 34a67f1..85aa2f9 --- a/sentry_sdk/serializer.py +++ b/sentry_sdk/serializer.py @@ -1,9 +1,11 @@ -import contextlib +import sys + from datetime import datetime from sentry_sdk.utils import ( AnnotatedValue, - capture_internal_exceptions, + capture_internal_exception, + disable_capture_event, safe_repr, strip_string, ) @@ -13,17 +15,20 @@ from sentry_sdk._types import MYPY if MYPY: + from types import TracebackType + from typing import Any from typing import Dict from typing import List from typing import Optional from typing import Callable from typing import Union - from typing import Generator + from typing import ContextManager + from typing import Type - # https://github.com/python/mypy/issues/5710 - _NotImplemented = Any - ReprProcessor = Callable[[Any, Dict[str, Any]], Union[_NotImplemented, str]] + from sentry_sdk._types import NotImplementedType, Event + + ReprProcessor = Callable[[Any, Dict[str, Any]], Union[NotImplementedType, str]] Segment = Union[str, int] @@ -31,11 +36,17 @@ # Importing ABCs from collections is deprecated, and will stop working in 3.8 # https://github.com/python/cpython/blob/master/Lib/collections/__init__.py#L49 from collections import Mapping, Sequence + + serializable_str_types = string_types + else: # New in 3.3 # https://docs.python.org/3/library/collections.abc.html from collections.abc import Mapping, Sequence + # Bytes are technically not strings in Python 3, but we can serialize them + serializable_str_types = (str, bytes) + MAX_DATABAG_DEPTH = 5 MAX_DATABAG_BREADTH = 10 CYCLE_MARKER = u"" @@ -49,248 +60,276 @@ def add_global_repr_processor(processor): global_repr_processors.append(processor) -class MetaNode(object): - __slots__ = ( - "_parent", - "_segment", - "_depth", - "_data", - "_is_databag", - "_should_repr_strings", - ) +class Memo(object): + __slots__ = ("_ids", "_objs") def __init__(self): # type: () -> None - self._parent = None # type: Optional[MetaNode] - self._segment = None # type: Optional[Segment] - self._depth = 0 # type: int - self._data = None # type: Optional[Dict[str, Any]] - self._is_databag = None # type: Optional[bool] - self._should_repr_strings = None # type: Optional[bool] - - def startswith_path(self, path): - # type: (List[Optional[str]]) -> bool - if len(path) > self._depth: - return False - - return self.is_path(path + [None] * (self._depth - len(path))) - - def is_path(self, path): - # type: (List[Optional[str]]) -> bool - if len(path) != self._depth: - return False - - cur = self - for segment in reversed(path): - if segment is not None and segment != cur._segment: - return False - assert cur._parent is not None - cur = cur._parent - - return cur._segment is None - - def enter(self, segment): - # type: (Segment) -> MetaNode - rv = MetaNode() - rv._parent = self - rv._depth = self._depth + 1 - rv._segment = segment - return rv - - def _create_annotations(self): - # type: () -> None - if self._data is not None: - return - - self._data = {} - if self._parent is not None: - self._parent._create_annotations() - self._parent._data[str(self._segment)] = self._data # type: ignore + self._ids = {} # type: Dict[int, Any] + self._objs = [] # type: List[Any] - def annotate(self, **meta): - # type: (Any) -> None - self._create_annotations() - assert self._data is not None - self._data.setdefault("", {}).update(meta) - - def should_repr_strings(self): - # type: () -> bool - if self._should_repr_strings is None: - self._should_repr_strings = ( - self.startswith_path( - ["exception", "values", None, "stacktrace", "frames", None, "vars"] - ) - or self.startswith_path( - ["threads", "values", None, "stacktrace", "frames", None, "vars"] - ) - or self.startswith_path(["stacktrace", "frames", None, "vars"]) - ) - - return self._should_repr_strings + def memoize(self, obj): + # type: (Any) -> ContextManager[bool] + self._objs.append(obj) + return self - def is_databag(self): + def __enter__(self): # type: () -> bool - if self._is_databag is None: - self._is_databag = ( - self.startswith_path(["request", "data"]) - or self.startswith_path(["breadcrumbs", None]) - or self.startswith_path(["extra"]) - or self.startswith_path( - ["exception", "values", None, "stacktrace", "frames", None, "vars"] - ) - or self.startswith_path( - ["threads", "values", None, "stacktrace", "frames", None, "vars"] - ) - or self.startswith_path(["stacktrace", "frames", None, "vars"]) - ) + obj = self._objs[-1] + if id(obj) in self._ids: + return True + else: + self._ids[id(obj)] = obj + return False - return self._is_databag + def __exit__( + self, + ty, # type: Optional[Type[BaseException]] + value, # type: Optional[BaseException] + tb, # type: Optional[TracebackType] + ): + # type: (...) -> None + self._ids.pop(id(self._objs.pop()), None) + + +def serialize(event, **kwargs): + # type: (Event, **Any) -> Event + memo = Memo() + path = [] # type: List[Segment] + meta_stack = [] # type: List[Dict[str, Any]] + + def _annotate(**meta): + # type: (**Any) -> None + while len(meta_stack) <= len(path): + try: + segment = path[len(meta_stack) - 1] + node = meta_stack[-1].setdefault(text_type(segment), {}) + except IndexError: + node = {} + + meta_stack.append(node) + + meta_stack[-1].setdefault("", {}).update(meta) + + def _should_repr_strings(): + # type: () -> Optional[bool] + """ + By default non-serializable objects are going through + safe_repr(). For certain places in the event (local vars) we + want to repr() even things that are JSON-serializable to + make their type more apparent. For example, it's useful to + see the difference between a unicode-string and a bytestring + when viewing a stacktrace. + + For container-types we still don't do anything different. + Generally we just try to make the Sentry UI present exactly + what a pretty-printed repr would look like. + + :returns: `True` if we are somewhere in frame variables, and `False` if + we are in a position where we will never encounter frame variables + when recursing (for example, we're in `event.extra`). `None` if we + are not (yet) in frame variables, but might encounter them when + recursing (e.g. we're in `event.exception`) + """ + try: + p0 = path[0] + if p0 == "stacktrace" and path[1] == "frames" and path[3] == "vars": + return True + + if ( + p0 in ("threads", "exception") + and path[1] == "values" + and path[3] == "stacktrace" + and path[4] == "frames" + and path[6] == "vars" + ): + return True + except IndexError: + return None + return False -def _flatten_annotated(obj, meta_node): - # type: (Any, MetaNode) -> Any - if isinstance(obj, AnnotatedValue): - meta_node.annotate(**obj.metadata) - obj = obj.value - return obj + def _is_databag(): + # type: () -> Optional[bool] + """ + A databag is any value that we need to trim. + :returns: Works like `_should_repr_strings()`. `True` for "yes", + `False` for :"no", `None` for "maybe soon". + """ + try: + rv = _should_repr_strings() + if rv in (True, None): + return rv -class Memo(object): - def __init__(self): - # type: () -> None - self._inner = {} # type: Dict[int, Any] + p0 = path[0] + if p0 == "request" and path[1] == "data": + return True - @contextlib.contextmanager - def memoize(self, obj): - # type: (Any) -> Generator[bool, None, None] - if id(obj) in self._inner: - yield True - else: - self._inner[id(obj)] = obj - yield False + if p0 == "breadcrumbs": + path[1] + return True - self._inner.pop(id(obj), None) + if p0 == "extra": + return True + except IndexError: + return None -class Serializer(object): - def __init__(self): - # type: () -> None - self.memo = Memo() - self.meta_node = MetaNode() + return False - @contextlib.contextmanager - def enter(self, segment): - # type: (Segment) -> Generator[None, None, None] - old_node = self.meta_node - self.meta_node = self.meta_node.enter(segment) + def _serialize_node( + obj, # type: Any + is_databag=None, # type: Optional[bool] + should_repr_strings=None, # type: Optional[bool] + segment=None, # type: Optional[Segment] + remaining_breadth=None, # type: Optional[int] + remaining_depth=None, # type: Optional[int] + ): + # type: (...) -> Any + if segment is not None: + path.append(segment) try: - yield - finally: - self.meta_node = old_node - - def serialize_event(self, obj): - # type: (Any) -> Dict[str, Any] - rv = self._serialize_node(obj) - if self.meta_node._data is not None: - rv["_meta"] = self.meta_node._data - return rv - - def _serialize_node(self, obj, max_depth=None, max_breadth=None): - # type: (Any, Optional[int], Optional[int]) -> Any - with capture_internal_exceptions(): - with self.memo.memoize(obj) as result: + with memo.memoize(obj) as result: if result: return CYCLE_MARKER - return self._serialize_node_impl( - obj, max_depth=max_depth, max_breadth=max_breadth + return _serialize_node_impl( + obj, + is_databag=is_databag, + should_repr_strings=should_repr_strings, + remaining_depth=remaining_depth, + remaining_breadth=remaining_breadth, ) + except BaseException: + capture_internal_exception(sys.exc_info()) - if self.meta_node.is_databag(): - return u"" - - return None - - def _serialize_node_impl(self, obj, max_depth, max_breadth): - # type: (Any, Optional[int], Optional[int]) -> Any - if max_depth is None and max_breadth is None and self.meta_node.is_databag(): - max_depth = self.meta_node._depth + MAX_DATABAG_DEPTH - max_breadth = self.meta_node._depth + MAX_DATABAG_BREADTH - - if max_depth is None: - remaining_depth = None - else: - remaining_depth = max_depth - self.meta_node._depth + if is_databag: + return u"" - obj = _flatten_annotated(obj, self.meta_node) + return None + finally: + if segment is not None: + path.pop() + del meta_stack[len(path) + 1 :] + + def _flatten_annotated(obj): + # type: (Any) -> Any + if isinstance(obj, AnnotatedValue): + _annotate(**obj.metadata) + obj = obj.value + return obj + + def _serialize_node_impl( + obj, is_databag, should_repr_strings, remaining_depth, remaining_breadth + ): + # type: (Any, Optional[bool], Optional[bool], Optional[int], Optional[int]) -> Any + if should_repr_strings is None: + should_repr_strings = _should_repr_strings() + + if is_databag is None: + is_databag = _is_databag() + + if is_databag and remaining_depth is None: + remaining_depth = MAX_DATABAG_DEPTH + if is_databag and remaining_breadth is None: + remaining_breadth = MAX_DATABAG_BREADTH + + obj = _flatten_annotated(obj) if remaining_depth is not None and remaining_depth <= 0: - self.meta_node.annotate(rem=[["!limit", "x"]]) - if self.meta_node.is_databag(): - return _flatten_annotated(strip_string(safe_repr(obj)), self.meta_node) + _annotate(rem=[["!limit", "x"]]) + if is_databag: + return _flatten_annotated(strip_string(safe_repr(obj))) return None - if self.meta_node.is_databag(): - hints = {"memo": self.memo, "remaining_depth": remaining_depth} + if is_databag and global_repr_processors: + hints = {"memo": memo, "remaining_depth": remaining_depth} for processor in global_repr_processors: - with capture_internal_exceptions(): - result = processor(obj, hints) - if result is not NotImplemented: - return _flatten_annotated(result, self.meta_node) + result = processor(obj, hints) + if result is not NotImplemented: + return _flatten_annotated(result) + + if obj is None or isinstance(obj, (bool, number_types)): + return obj if not should_repr_strings else safe_repr(obj) + + elif isinstance(obj, datetime): + return ( + text_type(obj.strftime("%Y-%m-%dT%H:%M:%S.%fZ")) + if not should_repr_strings + else safe_repr(obj) + ) - if isinstance(obj, Mapping): - # Create temporary list here to avoid calling too much code that + elif isinstance(obj, Mapping): + # Create temporary copy here to avoid calling too much code that # might mutate our dictionary while we're still iterating over it. - items = [] - for i, (k, v) in enumerate(iteritems(obj)): - if max_breadth is not None and i >= max_breadth: - self.meta_node.annotate(len=max_breadth) - break + obj = dict(iteritems(obj)) - items.append((k, v)) + rv_dict = {} # type: Dict[str, Any] + i = 0 - rv_dict = {} # type: Dict[Any, Any] - for k, v in items: - k = text_type(k) + for k, v in iteritems(obj): + if remaining_breadth is not None and i >= remaining_breadth: + _annotate(len=len(obj)) + break - with self.enter(k): - v = self._serialize_node( - v, max_depth=max_depth, max_breadth=max_breadth - ) - if v is not None: - rv_dict[k] = v + str_k = text_type(k) + v = _serialize_node( + v, + segment=str_k, + should_repr_strings=should_repr_strings, + is_databag=is_databag, + remaining_depth=remaining_depth - 1 + if remaining_depth is not None + else None, + remaining_breadth=remaining_breadth, + ) + rv_dict[str_k] = v + i += 1 return rv_dict - elif isinstance(obj, Sequence) and not isinstance(obj, string_types): - rv_list = [] # type: List[Any] + + elif not isinstance(obj, serializable_str_types) and isinstance(obj, Sequence): + rv_list = [] + for i, v in enumerate(obj): - if max_breadth is not None and i >= max_breadth: - self.meta_node.annotate(len=max_breadth) + if remaining_breadth is not None and i >= remaining_breadth: + _annotate(len=len(obj)) break - with self.enter(i): - rv_list.append( - self._serialize_node( - v, max_depth=max_depth, max_breadth=max_breadth - ) + rv_list.append( + _serialize_node( + v, + segment=i, + should_repr_strings=should_repr_strings, + is_databag=is_databag, + remaining_depth=remaining_depth - 1 + if remaining_depth is not None + else None, + remaining_breadth=remaining_breadth, ) + ) return rv_list - if self.meta_node.should_repr_strings(): + if should_repr_strings: obj = safe_repr(obj) else: - if obj is None or isinstance(obj, (bool, number_types)): - return obj - - if isinstance(obj, datetime): - return text_type(obj.strftime("%Y-%m-%dT%H:%M:%SZ")) - if isinstance(obj, bytes): obj = obj.decode("utf-8", "replace") if not isinstance(obj, string_types): obj = safe_repr(obj) - return _flatten_annotated(strip_string(obj), self.meta_node) + return _flatten_annotated(strip_string(obj)) + + disable_capture_event.set(True) + try: + rv = _serialize_node(event, **kwargs) + if meta_stack and isinstance(rv, dict): + rv["_meta"] = meta_stack[0] + + return rv + finally: + disable_capture_event.set(False) diff --git a/sentry_sdk/tracing.py b/sentry_sdk/tracing.py old mode 100755 new mode 100644 index ca1258a..cf971af --- a/sentry_sdk/tracing.py +++ b/sentry_sdk/tracing.py @@ -1,19 +1,29 @@ import re import uuid import contextlib -import collections from datetime import datetime -from sentry_sdk.utils import capture_internal_exceptions, concat_strings +import sentry_sdk +from sentry_sdk.utils import capture_internal_exceptions, logger, to_string +from sentry_sdk._compat import PY2 +from sentry_sdk._types import MYPY -if False: +if PY2: + from collections import Mapping +else: + from collections.abc import Mapping + +if MYPY: + import typing + + from typing import Generator from typing import Optional from typing import Any from typing import Dict - from typing import Mapping from typing import List + from typing import Tuple _traceparent_header_format_re = re.compile( "^[ \t]*" # whitespace @@ -24,10 +34,10 @@ ) -class EnvironHeaders(collections.Mapping): # type: ignore +class EnvironHeaders(Mapping): # type: ignore def __init__( self, - environ, # type: Mapping[str, str] + environ, # type: typing.Mapping[str, str] prefix="HTTP_", # type: str ): # type: (...) -> None @@ -35,12 +45,15 @@ def __init__( self.prefix = prefix def __getitem__(self, key): + # type: (str) -> Optional[Any] return self.environ[self.prefix + key.replace("-", "_").upper()] def __len__(self): + # type: () -> int return sum(1 for _ in iter(self)) def __iter__(self): + # type: () -> Generator[str, None, None] for k in self.environ: if not isinstance(k, str): continue @@ -52,6 +65,31 @@ def __iter__(self): yield k[len(self.prefix) :] +class _SpanRecorder(object): + __slots__ = ("maxlen", "finished_spans", "open_span_count") + + def __init__(self, maxlen): + # type: (int) -> None + self.maxlen = maxlen + self.open_span_count = 0 # type: int + self.finished_spans = [] # type: List[Span] + + def start_span(self, span): + # type: (Span) -> None + + # This is just so that we don't run out of memory while recording a lot + # of spans. At some point we just stop and flush out the start of the + # trace tree (i.e. the first n spans with the smallest + # start_timestamp). + self.open_span_count += 1 + if self.open_span_count > self.maxlen: + span._span_recorder = None + + def finish_span(self, span): + # type: (Span) -> None + self.finished_spans.append(span) + + class Span(object): __slots__ = ( "trace_id", @@ -66,20 +104,24 @@ class Span(object): "timestamp", "_tags", "_data", - "_finished_spans", + "_span_recorder", + "hub", + "_context_manager_state", ) def __init__( self, - trace_id=None, - span_id=None, - parent_span_id=None, - same_process_as_parent=True, - sampled=None, - transaction=None, - op=None, - description=None, + trace_id=None, # type: Optional[str] + span_id=None, # type: Optional[str] + parent_span_id=None, # type: Optional[str] + same_process_as_parent=True, # type: bool + sampled=None, # type: Optional[bool] + transaction=None, # type: Optional[str] + op=None, # type: Optional[str] + description=None, # type: Optional[str] + hub=None, # type: Optional[sentry_sdk.Hub] ): + # type: (...) -> None self.trace_id = trace_id or uuid.uuid4().hex self.span_id = span_id or uuid.uuid4().hex[16:] self.parent_span_id = parent_span_id @@ -88,15 +130,24 @@ def __init__( self.transaction = transaction self.op = op self.description = description + self.hub = hub self._tags = {} # type: Dict[str, str] self._data = {} # type: Dict[str, Any] - self._finished_spans = [] # type: List[Span] - self.start_timestamp = datetime.now() + self.start_timestamp = datetime.utcnow() #: End timestamp of span - self.timestamp = None + self.timestamp = None # type: Optional[datetime] + + self._span_recorder = None # type: Optional[_SpanRecorder] + + def init_finished_spans(self, maxlen): + # type: (int) -> None + if self._span_recorder is None: + self._span_recorder = _SpanRecorder(maxlen) + self._span_recorder.start_span(self) def __repr__(self): + # type: () -> str return ( "<%s(transaction=%r, trace_id=%r, span_id=%r, parent_span_id=%r, sampled=%r)>" % ( @@ -109,7 +160,29 @@ def __repr__(self): ) ) + def __enter__(self): + # type: () -> Span + hub = self.hub or sentry_sdk.Hub.current + + _, scope = hub._stack[-1] + old_span = scope.span + scope.span = self + self._context_manager_state = (hub, scope, old_span) + return self + + def __exit__(self, ty, value, tb): + # type: (Optional[Any], Optional[Any], Optional[Any]) -> None + if value is not None: + self._tags.setdefault("status", "internal_error") + + hub, scope, old_span = self._context_manager_state + del self._context_manager_state + + self.finish(hub) + scope.span = old_span + def new_span(self, **kwargs): + # type: (**Any) -> Span rv = type(self)( trace_id=self.trace_id, span_id=None, @@ -117,32 +190,38 @@ def new_span(self, **kwargs): sampled=self.sampled, **kwargs ) - rv._finished_spans = self._finished_spans + + rv._span_recorder = self._span_recorder return rv @classmethod def continue_from_environ(cls, environ): + # type: (typing.Mapping[str, str]) -> Span return cls.continue_from_headers(EnvironHeaders(environ)) @classmethod def continue_from_headers(cls, headers): + # type: (typing.Mapping[str, str]) -> Span parent = cls.from_traceparent(headers.get("sentry-trace")) if parent is None: return cls() - return parent.new_span(same_process_as_parent=False) + parent.same_process_as_parent = False + return parent def iter_headers(self): + # type: () -> Generator[Tuple[str, str], None, None] yield "sentry-trace", self.to_traceparent() @classmethod def from_traceparent(cls, traceparent): + # type: (Optional[str]) -> Optional[Span] if not traceparent: return None if traceparent.startswith("00-") and traceparent.endswith("-00"): traceparent = traceparent[3:-3] - match = _traceparent_header_format_re.match(traceparent) + match = _traceparent_header_format_re.match(str(traceparent)) if match is None: return None @@ -158,9 +237,10 @@ def from_traceparent(cls, traceparent): else: sampled = None - return cls(trace_id=trace_id, span_id=span_id, sampled=sampled) + return cls(trace_id=trace_id, parent_span_id=span_id, sampled=sampled) def to_traceparent(self): + # type: () -> str sampled = "" if self.sampled is True: sampled = "1" @@ -169,35 +249,143 @@ def to_traceparent(self): return "%s-%s-%s" % (self.trace_id, self.span_id, sampled) def to_legacy_traceparent(self): + # type: () -> str return "00-%s-%s-00" % (self.trace_id, self.span_id) def set_tag(self, key, value): + # type: (str, Any) -> None self._tags[key] = value def set_data(self, key, value): + # type: (str, Any) -> None self._data[key] = value - def finish(self): - self.timestamp = datetime.now() - self._finished_spans.append(self) + def set_status(self, value): + # type: (str) -> None + self.set_tag("status", value) + + def set_http_status(self, http_status): + # type: (int) -> None + self.set_tag("http.status_code", http_status) + + if http_status < 400: + self.set_status("ok") + elif 400 <= http_status < 500: + if http_status == 403: + self.set_status("permission_denied") + elif http_status == 404: + self.set_status("not_found") + elif http_status == 429: + self.set_status("resource_exhausted") + elif http_status == 413: + self.set_status("failed_precondition") + elif http_status == 401: + self.set_status("unauthenticated") + elif http_status == 409: + self.set_status("already_exists") + else: + self.set_status("invalid_argument") + elif 500 <= http_status < 600: + if http_status == 504: + self.set_status("deadline_exceeded") + elif http_status == 501: + self.set_status("unimplemented") + elif http_status == 503: + self.set_status("unavailable") + else: + self.set_status("internal_error") + else: + self.set_status("unknown_error") + + def is_success(self): + # type: () -> bool + return self._tags.get("status") == "ok" + + def finish(self, hub=None): + # type: (Optional[sentry_sdk.Hub]) -> Optional[str] + hub = hub or self.hub or sentry_sdk.Hub.current + + if self.timestamp is not None: + # This transaction is already finished, so we should not flush it again. + return None + + self.timestamp = datetime.utcnow() + + _maybe_create_breadcrumbs_from_span(hub, self) + + if self._span_recorder is None: + return None + + self._span_recorder.finish_span(self) + + if self.transaction is None: + # If this has no transaction set we assume there's a parent + # transaction for this span that would be flushed out eventually. + return None - def to_json(self): - return { + client = hub.client + + if client is None: + # We have no client and therefore nowhere to send this transaction + # event. + return None + + if not self.sampled: + # At this point a `sampled = None` should have already been + # resolved to a concrete decision. If `sampled` is `None`, it's + # likely that somebody used `with sentry_sdk.Hub.start_span(..)` on a + # non-transaction span and later decided to make it a transaction. + if self.sampled is None: + logger.warning("Discarding transaction Span without sampling decision") + + return None + + return hub.capture_event( + { + "type": "transaction", + "transaction": self.transaction, + "contexts": {"trace": self.get_trace_context()}, + "tags": self._tags, + "timestamp": self.timestamp, + "start_timestamp": self.start_timestamp, + "spans": [ + s.to_json(client) + for s in self._span_recorder.finished_spans + if s is not self + ], + } + ) + + def to_json(self, client): + # type: (Optional[sentry_sdk.Client]) -> Dict[str, Any] + rv = { "trace_id": self.trace_id, "span_id": self.span_id, "parent_span_id": self.parent_span_id, "same_process_as_parent": self.same_process_as_parent, - "transaction": self.transaction, "op": self.op, "description": self.description, "start_timestamp": self.start_timestamp, "timestamp": self.timestamp, - "tags": self._tags, - "data": self._data, - } + } # type: Dict[str, Any] + + transaction = self.transaction + if transaction: + rv["transaction"] = transaction + + tags = self._tags + if tags: + rv["tags"] = tags + + data = self._data + if data: + rv["data"] = data + + return rv def get_trace_context(self): - return { + # type: () -> Any + rv = { "trace_id": self.trace_id, "span_id": self.span_id, "parent_span_id": self.parent_span_id, @@ -205,57 +393,87 @@ def get_trace_context(self): "description": self.description, } + if "status" in self._tags: + rv["status"] = self._tags["status"] -@contextlib.contextmanager -def record_sql_queries(hub, queries, label=""): - if not queries: - yield None - else: - description = None - with capture_internal_exceptions(): - strings = [label] - for query in queries: - hub.add_breadcrumb(message=query, category="query") - strings.append(query) + return rv - description = concat_strings(strings) - if description is None: - yield None - else: - with hub.span(op="db", description=description) as span: - yield span +def _format_sql(cursor, sql): + # type: (Any, str) -> Optional[str] + + real_sql = None + + # If we're using psycopg2, it could be that we're + # looking at a query that uses Composed objects. Use psycopg2's mogrify + # function to format the query. We lose per-parameter trimming but gain + # accuracy in formatting. + try: + if hasattr(cursor, "mogrify"): + real_sql = cursor.mogrify(sql) + if isinstance(real_sql, bytes): + real_sql = real_sql.decode(cursor.connection.encoding) + except Exception: + real_sql = None + + return real_sql or to_string(sql) @contextlib.contextmanager -def record_http_request(hub, url, method): - data_dict = {"url": url, "method": method} +def record_sql_queries( + hub, # type: sentry_sdk.Hub + cursor, # type: Any + query, # type: Any + params_list, # type: Any + paramstyle, # type: Optional[str] + executemany, # type: bool +): + # type: (...) -> Generator[Span, None, None] + + # TODO: Bring back capturing of params by default + if hub.client and hub.client.options["_experiments"].get( + "record_sql_params", False + ): + if not params_list or params_list == [None]: + params_list = None + + if paramstyle == "pyformat": + paramstyle = "format" + else: + params_list = None + paramstyle = None + + query = _format_sql(cursor, query) - with hub.span(op="http", description="%s %s" % (url, method)) as span: - try: - yield data_dict - finally: - if span is not None: - if "status_code" in data_dict: - span.set_tag("http.status_code", data_dict["status_code"]) - for k, v in data_dict.items(): - span.set_data(k, v) + data = {} + if params_list is not None: + data["db.params"] = params_list + if paramstyle is not None: + data["db.paramstyle"] = paramstyle + if executemany: + data["db.executemany"] = True + with capture_internal_exceptions(): + hub.add_breadcrumb(message=query, category="query", data=data) -def maybe_create_breadcrumbs_from_span(hub, span): + with hub.start_span(op="db", description=query) as span: + for k, v in data.items(): + span.set_data(k, v) + yield span + + +def _maybe_create_breadcrumbs_from_span(hub, span): + # type: (sentry_sdk.Hub, Span) -> None if span.op == "redis": - hub.add_breadcrumb(type="redis", category="redis", data=span._tags) - elif span.op == "http" and not span._tags.get("error"): hub.add_breadcrumb( - type="http", - category="httplib", - data=span._data, - hint={"httplib_response": span._data.get("httplib_response")}, + message=span.description, type="redis", category="redis", data=span._tags ) + elif span.op == "http": + hub.add_breadcrumb(type="http", category="httplib", data=span._data) elif span.op == "subprocess": hub.add_breadcrumb( type="subprocess", category="subprocess", + message=span.description, data=span._data, - hint={"popen_instance": span._data.get("popen_instance")}, ) diff --git a/sentry_sdk/transport.py b/sentry_sdk/transport.py old mode 100755 new mode 100644 diff --git a/sentry_sdk/utils.py b/sentry_sdk/utils.py old mode 100755 new mode 100644 index 63e6b86..a9cac5d --- a/sentry_sdk/utils.py +++ b/sentry_sdk/utils.py @@ -3,27 +3,27 @@ import linecache import logging -from contextlib import contextmanager from datetime import datetime -from sentry_sdk._compat import urlparse, text_type, implements_str, int_types, PY2 +import sentry_sdk +from sentry_sdk._compat import urlparse, text_type, implements_str, PY2 from sentry_sdk._types import MYPY if MYPY: + from types import FrameType + from types import TracebackType from typing import Any from typing import Callable from typing import Dict + from typing import ContextManager from typing import Iterator from typing import List from typing import Optional from typing import Set from typing import Tuple from typing import Union - from types import FrameType - from types import TracebackType - - import sentry_sdk + from typing import Type from sentry_sdk._types import ExcInfo @@ -43,15 +43,34 @@ def _get_debug_hub(): pass -@contextmanager +class CaptureInternalException(object): + __slots__ = () + + def __enter__(self): + # type: () -> ContextManager[Any] + return self + + def __exit__(self, ty, value, tb): + # type: (Optional[Type[BaseException]], Optional[BaseException], Optional[TracebackType]) -> bool + if ty is not None and value is not None: + capture_internal_exception((ty, value, tb)) + + return True + + +_CAPTURE_INTERNAL_EXCEPTION = CaptureInternalException() + + def capture_internal_exceptions(): - # type: () -> Iterator - try: - yield - except Exception: - hub = _get_debug_hub() - if hub is not None: - hub._capture_internal_exception(sys.exc_info()) + # type: () -> ContextManager[Any] + return _CAPTURE_INTERNAL_EXCEPTION + + +def capture_internal_exception(exc_info): + # type: (ExcInfo) -> None + hub = _get_debug_hub() + if hub is not None: + hub._capture_internal_exception(exc_info) def to_timestamp(value): @@ -85,16 +104,25 @@ def __init__(self, value): self.__dict__ = dict(value.__dict__) return parts = urlparse.urlsplit(text_type(value)) + if parts.scheme not in (u"http", u"https"): raise BadDsn("Unsupported scheme %r" % parts.scheme) self.scheme = parts.scheme + + if parts.hostname is None: + raise BadDsn("Missing hostname") + self.host = parts.hostname - self.port = parts.port - if self.port is None: + + if parts.port is None: self.port = self.scheme == "https" and 443 or 80 - self.public_key = parts.username - if not self.public_key: + else: + self.port = parts.port + + if not parts.username: raise BadDsn("Missing public key") + + self.public_key = parts.username self.secret_key = parts.password path = parts.path.rsplit("/", 1) @@ -189,12 +217,21 @@ def to_header(self, timestamp=None): class AnnotatedValue(object): + __slots__ = ("value", "metadata") + def __init__(self, value, metadata): # type: (Optional[Any], Dict[str, Any]) -> None self.value = value self.metadata = metadata +if MYPY: + from typing import TypeVar + + T = TypeVar("T") + Annotated = Union[AnnotatedValue, T] + + def get_type_name(cls): # type: (Optional[type]) -> Optional[str] return getattr(cls, "__qualname__", None) or getattr(cls, "__name__", None) @@ -236,22 +273,13 @@ def iter_stacks(tb): tb_ = tb_.tb_next -def slim_string(value, length=MAX_STRING_LENGTH): - # type: (str, int) -> str - if not value: - return value - if len(value) > length: - return value[: length - 3] + "..." - return value[:length] - - def get_lines_from_file( filename, # type: str lineno, # type: int loader=None, # type: Optional[Any] module=None, # type: Optional[str] ): - # type: (...) -> Tuple[List[str], Optional[str], List[str]] + # type: (...) -> Tuple[List[Annotated[str]], Optional[Annotated[str]], List[Annotated[str]]] context_lines = 5 source = None if loader is not None and hasattr(loader, "get_source"): @@ -276,11 +304,11 @@ def get_lines_from_file( try: pre_context = [ - slim_string(line.strip("\r\n")) for line in source[lower_bound:lineno] + strip_string(line.strip("\r\n")) for line in source[lower_bound:lineno] ] - context_line = slim_string(source[lineno].strip("\r\n")) + context_line = strip_string(source[lineno].strip("\r\n")) post_context = [ - slim_string(line.strip("\r\n")) + strip_string(line.strip("\r\n")) for line in source[(lineno + 1) : upper_bound] ] return pre_context, context_line, post_context @@ -289,8 +317,11 @@ def get_lines_from_file( return [], None, [] -def get_source_context(frame, tb_lineno): - # type: (FrameType, int) -> Tuple[List[str], Optional[str], List[str]] +def get_source_context( + frame, # type: FrameType + tb_lineno, # type: int +): + # type: (...) -> Tuple[List[Annotated[str]], Optional[Annotated[str]], List[Annotated[str]]] try: abs_path = frame.f_code.co_filename # type: Optional[str] except Exception: @@ -317,32 +348,42 @@ def safe_str(value): return safe_repr(value) -def safe_repr(value): - # type: (Any) -> str - try: - rv = repr(value) - if isinstance(rv, bytes): - rv = rv.decode("utf-8", "replace") - - # At this point `rv` contains a bunch of literal escape codes, like - # this (exaggerated example): - # - # u"\\x2f" - # - # But we want to show this string as: - # - # u"/" +if PY2: + + def safe_repr(value): + # type: (Any) -> str try: - # unicode-escape does this job, but can only decode latin1. So we - # attempt to encode in latin1. - return rv.encode("latin1").decode("unicode-escape") + rv = repr(value).decode("utf-8", "replace") + + # At this point `rv` contains a bunch of literal escape codes, like + # this (exaggerated example): + # + # u"\\x2f" + # + # But we want to show this string as: + # + # u"/" + try: + # unicode-escape does this job, but can only decode latin1. So we + # attempt to encode in latin1. + return rv.encode("latin1").decode("unicode-escape") + except Exception: + # Since usually strings aren't latin1 this can break. In those + # cases we just give up. + return rv except Exception: - # Since usually strings aren't latin1 this can break. In those - # cases we just give up. - return rv - except Exception: - # If e.g. the call to `repr` already fails - return u"" + # If e.g. the call to `repr` already fails + return u"" + + +else: + + def safe_repr(value): + # type: (Any) -> str + try: + return repr(value) + except Exception: + return "" def filename_for_module(module, abs_path): @@ -397,6 +438,7 @@ def serialize_frame(frame, tb_lineno=None, with_locals=True): } # type: Dict[str, Any] if with_locals: rv["vars"] = frame.f_locals + return rv @@ -437,7 +479,7 @@ def single_exception_from_error_tuple( exc_type, # type: Optional[type] exc_value, # type: Optional[BaseException] tb, # type: Optional[TracebackType] - client_options=None, # type: Optional[dict] + client_options=None, # type: Optional[Dict[str, Any]] mechanism=None, # type: Optional[Dict[str, Any]] ): # type: (...) -> Dict[str, Any] @@ -510,7 +552,7 @@ def walk_exception_chain(exc_info): def exceptions_from_error_tuple( exc_info, # type: ExcInfo - client_options=None, # type: Optional[dict] + client_options=None, # type: Optional[Dict[str, Any]] mechanism=None, # type: Optional[Dict[str, Any]] ): # type: (...) -> List[Dict[str, Any]] @@ -558,7 +600,7 @@ def iter_event_frames(event): def handle_in_app(event, in_app_exclude=None, in_app_include=None): - # type: (Dict[str, Any], Optional[List], Optional[List]) -> Dict[str, Any] + # type: (Dict[str, Any], Optional[List[str]], Optional[List[str]]) -> Dict[str, Any] for stacktrace in iter_event_stacktraces(event): handle_in_app_impl( stacktrace.get("frames"), @@ -570,7 +612,7 @@ def handle_in_app(event, in_app_exclude=None, in_app_include=None): def handle_in_app_impl(frames, in_app_exclude, in_app_include): - # type: (Any, Optional[List], Optional[List]) -> Optional[Any] + # type: (Any, Optional[List[str]], Optional[List[str]]) -> Optional[Any] if not frames: return None @@ -623,7 +665,7 @@ def exc_info_from_error(error): def event_from_exception( exc_info, # type: Union[BaseException, ExcInfo] - client_options=None, # type: Optional[dict] + client_options=None, # type: Optional[Dict[str, Any]] mechanism=None, # type: Optional[Dict[str, Any]] ): # type: (...) -> Tuple[Dict[str, Any], Dict[str, Any]] @@ -643,7 +685,7 @@ def event_from_exception( def _module_in_set(name, set): - # type: (str, Optional[List]) -> bool + # type: (str, Optional[List[str]]) -> bool if not set: return False for item in set or (): @@ -652,12 +694,18 @@ def _module_in_set(name, set): return False -def strip_string(value, max_length=512): - # type: (str, int) -> Union[AnnotatedValue, str] +def strip_string(value, max_length=None): + # type: (str, Optional[int]) -> Union[AnnotatedValue, str] # TODO: read max_length from config if not value: return value + + if max_length is None: + # This is intentionally not just the default such that one can patch `MAX_STRING_LENGTH` and affect `strip_string`. + max_length = MAX_STRING_LENGTH + length = len(value) + if length > max_length: return AnnotatedValue( value=value[: max_length - 3] + u"...", @@ -669,97 +717,12 @@ def strip_string(value, max_length=512): return value -def format_and_strip( - template, params, strip_string=strip_string, max_length=MAX_FORMAT_PARAM_LENGTH -): - """Format a string containing %s for placeholders and call `strip_string` - on each parameter. The string template itself does not have a maximum - length. - - TODO: handle other placeholders, not just %s - """ - chunks = template.split(u"%s") - if not chunks: - raise ValueError("No formatting placeholders found") - - params = params[: len(chunks) - 1] - - if len(params) < len(chunks) - 1: - raise ValueError("Not enough params.") - - concat_chunks = [] - iter_chunks = iter(chunks) # type: Optional[Iterator] - iter_params = iter(params) # type: Optional[Iterator] - - while iter_chunks is not None or iter_params is not None: - if iter_chunks is not None: - try: - concat_chunks.append(next(iter_chunks)) - except StopIteration: - iter_chunks = None - - if iter_params is not None: - try: - concat_chunks.append(str(next(iter_params))) - except StopIteration: - iter_params = None - - return concat_strings( - concat_chunks, strip_string=strip_string, max_length=max_length - ) - - -def concat_strings( - chunks, strip_string=strip_string, max_length=MAX_FORMAT_PARAM_LENGTH -): - rv_remarks = [] # type: List[Any] - rv_original_length = 0 - rv_length = 0 - rv = [] # type: List[str] - - def realign_remark(remark): - return [ - (rv_length + x if isinstance(x, int_types) and i < 4 else x) - for i, x in enumerate(remark) - ] - - for chunk in chunks: - if isinstance(chunk, AnnotatedValue): - # Assume it's already stripped! - stripped_chunk = chunk - chunk = chunk.value - else: - stripped_chunk = strip_string(chunk, max_length=max_length) - - if isinstance(stripped_chunk, AnnotatedValue): - rv_remarks.extend( - realign_remark(remark) for remark in stripped_chunk.metadata["rem"] - ) - stripped_chunk_value = stripped_chunk.value - else: - stripped_chunk_value = stripped_chunk - - rv_original_length += len(chunk) - rv_length += len(stripped_chunk_value) # type: ignore - rv.append(stripped_chunk_value) # type: ignore - - rv_joined = u"".join(rv) - assert len(rv_joined) == rv_length - - if not rv_remarks: - return rv_joined - - return AnnotatedValue( - value=rv_joined, metadata={"len": rv_original_length, "rem": rv_remarks} - ) - - def _is_threading_local_monkey_patched(): # type: () -> bool try: from gevent.monkey import is_object_patched # type: ignore - if is_object_patched("_threading", "local"): + if is_object_patched("threading", "local"): return True except ImportError: pass @@ -775,12 +738,8 @@ def _is_threading_local_monkey_patched(): return False -IS_THREADING_LOCAL_MONKEY_PATCHED = _is_threading_local_monkey_patched() -del _is_threading_local_monkey_patched - - def _get_contextvars(): - # () -> (bool, Type) + # type: () -> Tuple[bool, type] """ Try to import contextvars and use it if it's deemed safe. We should not use contextvars if gevent or eventlet have patched thread locals, as @@ -788,12 +747,12 @@ def _get_contextvars(): https://github.com/gevent/gevent/issues/1407 """ - if not IS_THREADING_LOCAL_MONKEY_PATCHED: + if not _is_threading_local_monkey_patched(): try: - from contextvars import ContextVar # type: ignore + from contextvars import ContextVar if not PY2 and sys.version_info < (3, 7): - import aiocontextvars # type: ignore # noqa + import aiocontextvars # noqa return True, ContextVar except ImportError: @@ -801,24 +760,26 @@ def _get_contextvars(): from threading import local - class ContextVar(object): # type: ignore + class ContextVar(object): # Super-limited impl of ContextVar def __init__(self, name): + # type: (str) -> None self._name = name self._local = local() def get(self, default): + # type: (Any) -> Any return getattr(self._local, "value", default) def set(self, value): - setattr(self._local, "value", value) + # type: (Any) -> None + self._local.value = value return False, ContextVar HAS_REAL_CONTEXTVARS, ContextVar = _get_contextvars() -del _get_contextvars def transaction_from_function(func): @@ -851,3 +812,6 @@ def transaction_from_function(func): # Possibly a lambda return func_qualname + + +disable_capture_event = ContextVar("disable_capture_event") diff --git a/sentry_sdk/worker.py b/sentry_sdk/worker.py old mode 100755 new mode 100644 index 92ba8f1..0efcc68 --- a/sentry_sdk/worker.py +++ b/sentry_sdk/worker.py @@ -45,16 +45,33 @@ def _timed_queue_join(self, timeout): # type: (float) -> bool deadline = time() + timeout queue = self._queue - queue.all_tasks_done.acquire() # type: ignore + + real_all_tasks_done = getattr( + queue, "all_tasks_done", None + ) # type: Optional[Any] + if real_all_tasks_done is not None: + real_all_tasks_done.acquire() + all_tasks_done = real_all_tasks_done # type: Optional[Any] + elif queue.__module__.startswith("eventlet."): + all_tasks_done = getattr(queue, "_cond", None) + else: + all_tasks_done = None + try: - while queue.unfinished_tasks: # type: ignore + while queue.unfinished_tasks: delay = deadline - time() if delay <= 0: return False - queue.all_tasks_done.wait(timeout=delay) # type: ignore + if all_tasks_done is not None: + all_tasks_done.wait(timeout=delay) + else: + # worst case, we just poll the number of remaining tasks + sleep(0.1) + return True finally: - queue.all_tasks_done.release() # type: ignore + if real_all_tasks_done is not None: + real_all_tasks_done.release() def start(self): # type: () -> None