diff --git a/build/update_version_from_git.py b/build/update_version_from_git.py index 9b54e6c491e..48e4c24a944 100755 --- a/build/update_version_from_git.py +++ b/build/update_version_from_git.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import logging +import os import sys from os import linesep, path from subprocess import PIPE, Popen @@ -28,13 +29,24 @@ def run_command(cmd): commit_id = run_command(cmd).strip()[1:].replace("'", "") logger.info("Commit: %s", commit_id) + sentry_url = os.environ.get('SENTRY_URL', None) + logger.info(f'Sentry url: {sentry_url}') + if not sentry_url: + logger.critical('Sentry url is not defined. To define sentry url use:' + 'EXPORT SENTRY_URL=') + sys.exit(1) + build_date = ctime() logger.info("Build date: %s", build_date) logger.info('Writing runtime version info.') with open(path.join('src', 'tribler-core', 'tribler_core', 'version.py'), 'w') as f: - f.write('version_id = "%s"%sbuild_date = "%s"%scommit_id = "%s"%s' % - (version_id, linesep, build_date, linesep, commit_id, linesep)) + f.write( + f'version_id = "{version_id}"{linesep}' + f'build_date = "{build_date}"{linesep}' + f'commit_id = "{commit_id}"{linesep}' + f'sentry_url = "{sentry_url}"{linesep}' + ) with open('.TriblerVersion', 'w') as f: f.write(version_id) diff --git a/src/README.md b/src/README.md index 870bff53480..8618c2bb770 100644 --- a/src/README.md +++ b/src/README.md @@ -25,8 +25,13 @@ Export to PYTHONPATH the following directories: * tribler-core * tribler-gui +Shortcut for macOS: +```shell script +export PYTHONPATH=${PYTHONPATH}:`echo {pyipv8,tribler-common,tribler-core,tribler-gui} | tr " " :` +``` Execute: ``` python3 -m pytest tribler-core +python3 -m pytest tribler-common python3 -m pytest tribler-gui --guitests ``` \ No newline at end of file diff --git a/src/requirements.txt b/src/requirements.txt index 9449da81d8e..82f39f0d2db 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -15,6 +15,8 @@ aiohttp aiohttp_apispec pyyaml marshmallow +sentry_sdk +Faker Pillow netifaces pyqtgraph diff --git a/src/run_tribler.py b/src/run_tribler.py index 6c0b986aa36..9fb27012719 100644 --- a/src/run_tribler.py +++ b/src/run_tribler.py @@ -5,12 +5,15 @@ import sys from asyncio import ensure_future, get_event_loop +from tribler_common.sentry_reporter.sentry_reporter import SentryReporter +from tribler_common.sentry_reporter.sentry_scrubber import SentryScrubber + import tribler_core from tribler_core.config.tribler_config import CONFIG_FILENAME from tribler_core.dependencies import check_for_missing_dependencies from tribler_core.upgrade.version_manager import fork_state_directory_if_necessary, get_versioned_state_directory from tribler_core.utilities.osutils import get_root_state_directory -from tribler_core.version import version_id +from tribler_core.version import sentry_url, version_id import tribler_gui @@ -86,6 +89,9 @@ async def start_tribler(): if __name__ == "__main__": + SentryReporter.init(sentry_url=sentry_url, scrubber=SentryScrubber()) + SentryReporter.allow_sending_globally(False, 'run_tribler.__main__()') + # Get root state directory (e.g. from environment variable or from system default) root_state_dir = get_root_state_directory() diff --git a/src/tribler-common/tribler_common/sentry_reporter/sentry_reporter.py b/src/tribler-common/tribler_common/sentry_reporter/sentry_reporter.py new file mode 100644 index 00000000000..e969b5246c1 --- /dev/null +++ b/src/tribler-common/tribler_common/sentry_reporter/sentry_reporter.py @@ -0,0 +1,330 @@ +import logging +from contextvars import ContextVar +from hashlib import md5 + +from faker import Faker + +import sentry_sdk +from sentry_sdk.integrations.logging import LoggingIntegration +from sentry_sdk.integrations.threading import ThreadingIntegration + +from tribler_common.sentry_reporter.sentry_tools import ( + delete_item, + get_first_item, + get_value, + parse_os_environ, + parse_stacktrace, +) + +PLATFORM_DETAILS = 'platform.details' +STACKTRACE = '_stacktrace' +SYSINFO = 'sysinfo' +OS_ENVIRON = 'os.environ' +SYS_ARGV = 'sys.argv' +TAGS = 'tags' +CONTEXTS = 'contexts' +EXTRA = 'extra' +BREADCRUMBS = 'breadcrumbs' +LOGENTRY = 'logentry' +REPORTER = 'reporter' + + +class SentryReporter: + """SentryReporter designed for sending reports to the Sentry server from + a Tribler Client. + + # Main concept + + It aims to work in a Tribler Client with the following specifics: + + 1. An exception can be raised in two separate processes: GUI and Core. + 2. A report will be sent after a user presses the "send report" button. + + The main concept behind this class is the following: let's add controls that + can allow or disallow sending Sentry reports by-default. + + If we have these controls, then it is easy to use Sentry with a Tribler + Client. + + Algorithm: + 1. Initialise Sentry. + 2. Disallow sending messages (but storing the last event). + 3. Waiting for user action: the "sent report" button is pressed. + 4. Allow Sentry sending messages + 5. Send the last event. + + SentryReporter is thread-safe. + """ + + last_event = None + + _allow_sending_global = False + _allow_sending_in_thread = ContextVar('Sentry') + + _scrubber = None + _logger = logging.getLogger('SentryReporter') + + @staticmethod + def init(sentry_url='', scrubber=None): + """ Initialization. + + This method should be called in each process that uses SentryReporter. + + Args: + sentry_url: URL for Sentry server. If it is empty then Sentry's + sending mechanism will not be initialized. + + scrubber: a class that will be used for scrubbing sending events. + Only a single method should be implemented in the class: + ``` + def scrub_event(self, event): + pass + ``` + Returns: + Sentry Guard. + """ + SentryReporter._logger.info(f"Init: {sentry_url}") + + SentryReporter._scrubber = scrubber + return sentry_sdk.init( + sentry_url, + release=None, + # https://docs.sentry.io/platforms/python/configuration/integrations/ + integrations=[ + LoggingIntegration( + level=logging.INFO, # Capture info and above as breadcrumbs + event_level=logging.ERROR, # Send errors as events + ), + ThreadingIntegration(propagate_hub=True), + ], + before_send=SentryReporter._before_send, + ) + + @staticmethod + def send(event, post_data, sys_info): + """Send the event to the Sentry server + + This method + 1. Enable Sentry's sending mechanism. + 2. Extend sending event by the information from post_data. + 3. Send the event. + 4. Disables Sentry's sending mechanism. + + Scrubbing the information will be performed in the `_before_send` method. + + During the execution of this method, all unhandled exceptions that + will be raised, will be sent to Sentry automatically. + + Args: + event: event to send. It should be taken from SentryReporter at + post_data: dictionary made by the feedbackdialog.py + previous stages of executing. + sys_info: dictionary made by the feedbackdialog.py + + Returns: + Event that was sent to Sentry server + """ + SentryReporter._logger.info(f"Send: {post_data}, {event}") + + if event is None: + return event + + with AllowSentryReports(value=True, description='SentryReporter.send()'): + # prepare event + if CONTEXTS not in event: + event[CONTEXTS] = {} + + if TAGS not in event: + event[TAGS] = {} + + event[CONTEXTS][REPORTER] = {} + + # tags + tags = event[TAGS] + tags['version'] = get_value(post_data, 'version') + tags['machine'] = get_value(post_data, 'machine') + tags['os'] = get_value(post_data, 'os') + tags['platform'] = get_first_item(get_value(sys_info, 'platform')) + tags[('%s' % PLATFORM_DETAILS)] = get_first_item(get_value(sys_info, PLATFORM_DETAILS)) + + # context + context = event[CONTEXTS] + reporter = context[REPORTER] + version = get_value(post_data, 'version') + + context['browser'] = {'version': version, 'name': 'Tribler'} + + reporter[STACKTRACE] = parse_stacktrace(get_value(post_data, 'stack')) + reporter['comments'] = get_value(post_data, 'comments') + + reporter[OS_ENVIRON] = parse_os_environ(get_value(sys_info, OS_ENVIRON)) + delete_item(sys_info, OS_ENVIRON) + reporter[SYSINFO] = sys_info + + sentry_sdk.capture_event(event) + + return event + + @staticmethod + def set_user(user_id): + """ Set the user to identify the event on a Sentry server + + The algorithm is the following: + 1. Calculate hash from `user_id`. + 2. Generate fake user, based on the hash. + + No real `user_id` will be used in Sentry. + + Args: + user_id: Real user id. + + Returns: + Generated user (dictionary: {id, username}). + """ + # calculate hash to keep real `user_id` in secret + user_id_hash = md5(user_id).hexdigest() + + SentryReporter._logger.info(f"Set user: {user_id_hash}") + + Faker.seed(user_id_hash) + user_name = Faker().name() + user = {'id': user_id_hash, 'username': user_name} + + sentry_sdk.set_user(user) + return user + + @staticmethod + def get_allow_sending(): + """ Indicate whether Sentry allowed or disallowed to sent events. + + Returns: + Bool + """ + allow_sending_in_thread = SentryReporter._allow_sending_in_thread.get(None) + if allow_sending_in_thread is not None: + return allow_sending_in_thread + + return SentryReporter._allow_sending_global + + @staticmethod + def allow_sending_globally(value, info=None): + """ Setter for `_allow_sending_global` variable. + + It globally allows or disallows Sentry to send events. + If `_allow_sending_in_thread` is not set, then `_allow_sending_global` + will be used. + + Args: + value: Bool + info: String that will be used as an indicator of allowing or + disallowing reason (or the place from which this method has been + invoked). + + Returns: + None + """ + SentryReporter._logger.info(f"Allow sending globally: {value}. Info: {info}") + SentryReporter._allow_sending_global = value + + @staticmethod + def allow_sending_in_thread(value, info=None): + """ Setter for `_allow_sending` variable. + + It allows or disallows Sentry to send events in the current thread. + If `_allow_sending_in_thread` is not set, then `_allow_sending_global` + will be used. + + Args: + value: Bool + info: String that will be used as an indicator of allowing or + disallowing reason (or the place from which this method has been + invoked). + + Returns: + None + """ + SentryReporter._logger.info(f"Allow sending in thread: {value}. Info: {info}") + SentryReporter._allow_sending_in_thread.set(value) + + @staticmethod + def _before_send(event, _): + """The method that is called before each send. Both allowed and + disallowed. + + The algorithm: + 1. If sending is allowed, then scrub the event and send. + 2. If sending is disallowed, then store the event in + `SentryReporter.last_event` + + Args: + event: event that generated by Sentry + hint: root exception (can be used in some cases) + + Returns: + The event, prepared for sending, or `None`, if sending is suppressed. + """ + if not event: + return event + + SentryReporter._logger.info(f"Before send event: {event}") + SentryReporter._logger.info(f"Is allow sending: {SentryReporter._allow_sending_global}") + # to synchronise error reporter and sentry, we should suppress all events + # until user clicked on "send crash report" + if not SentryReporter.get_allow_sending(): + SentryReporter._logger.info("Suppress sending. Storing the event.") + SentryReporter.last_event = event + return None + + # clean up the event + SentryReporter._logger.info(f"Clean up the event with scrubber: {SentryReporter._scrubber}") + if SentryReporter._scrubber: + event = SentryReporter._scrubber.scrub_event(event) + + return event + + +class AllowSentryReports: + """ This class designed for simplifying allowing and disallowing + Sentry's sending mechanism for particular blocks of code. + + It is thread-safe, and use `SentryReporter.allow_sending_in_thread` method + for setting corresponding variable. + + Example of use: + ``` + with AllowSentryReports(value=True): + do_some_work() + ``` + """ + + def __init__(self, value=True, description='', reporter=None): + """ Initialising a value and a reporter + + Args: + value: Value that will be used for passing in + `SentryReporter.allow_sending`. + description: Will be used while logging. + reporter: Instance of a reporter. This argument mostly use for + testing purposes. + """ + self._logger = logging.getLogger(self.__class__.__name__) + self._logger.info(f'Value: {value}, description: {description}') + + self._value = value + self._saved_state = None + self._reporter = reporter or SentryReporter() + + def __enter__(self): + """Set SentryReporter.allow_sending(value) + """ + self._logger.info('Enter') + self._saved_state = self._reporter.get_allow_sending() + + self._reporter.allow_sending_in_thread(self._value, 'AllowSentryReports.__enter__()') + return self._reporter + + def __exit__(self, exc_type, exc_val, exc_tb): + """Restore SentryReporter.allow_sending(old_value) + """ + self._logger.info('Exit') + self._reporter.allow_sending_in_thread(self._saved_state, 'AllowSentryReports.__exit__()') diff --git a/src/tribler-common/tribler_common/sentry_reporter/sentry_scrubber.py b/src/tribler-common/tribler_common/sentry_reporter/sentry_scrubber.py new file mode 100644 index 00000000000..19cdbd31368 --- /dev/null +++ b/src/tribler-common/tribler_common/sentry_reporter/sentry_scrubber.py @@ -0,0 +1,165 @@ +import re + +from tribler_common.sentry_reporter.sentry_reporter import ( + BREADCRUMBS, + CONTEXTS, + EXTRA, + LOGENTRY, + OS_ENVIRON, + REPORTER, + STACKTRACE, + SYSINFO, +) +from tribler_common.sentry_reporter.sentry_tools import delete_item, modify_value + + +class SentryScrubber: + """ This class has been created to be responsible for scrubbing all sensitive + and unnecessary information from Sentry event. + """ + + def __init__(self): + # https://en.wikipedia.org/wiki/Home_directory + self.home_folders = [ + 'users', + 'usr', + 'home', + 'u01', + 'var', + r'data\/media', + r'WINNT\\Profiles', + 'Documents and Settings', + ] + + self.event_fields_to_cut = ['modules'] + + self.placeholder_user = '' + self.placeholder_ip = '' + self.placeholder_hash = '' + + self.exclusions = ['local', '127.0.0.1'] + + self.user_name = None + + self.re_folders = [] + self.re_ip = None + self.re_hash = None + + self._compile_re() + + def _compile_re(self): + """ Compile all regular expressions. + """ + for folder in self.home_folders: + folder_pattern = r'(?<=' + folder + r'[/\\])\w+(?=[/\\])' + self.re_folders.append(re.compile(folder_pattern, re.I)) + + self.re_ip = re.compile(r'(?>) + # but it is not really important what placeholder we use + # hence, let's leave it at that for now. + assert scrubber.scrub_text('/users/user/apps') == \ + f'/users/<{scrubber.placeholder_user}>/apps' + assert scrubber.user_name == 'user' + + assert scrubber.scrub_text('/users/username/some/long_path') == \ + f'/users/{scrubber.placeholder_user}/some/long_path' + assert scrubber.user_name == 'username' + + +def test_scrub_text_ip(scrubber): + # negative + assert scrubber.scrub_text('127.0.0.1') == '127.0.0.1' + assert scrubber.scrub_text('0.0.0') == '0.0.0' + + # positive + assert scrubber.scrub_text('0.0.0.1') == scrubber.placeholder_ip + assert scrubber.scrub_text('0.100.0.1') == scrubber.placeholder_ip + + assert scrubber.user_name is None + + +def test_scrub_text_hash(scrubber): + # negative + assert scrubber.scrub_text('0a303030303030303030303030303030303030300') == \ + '0a303030303030303030303030303030303030300' + assert scrubber.scrub_text('0a3030303030303030303303030303030303030') == \ + '0a3030303030303030303303030303030303030' + + # positive + assert scrubber.scrub_text('3030303030303030303030303030303030303030') == \ + scrubber.placeholder_hash + assert scrubber.scrub_text('hash:3030303030303030303030303030303030303030') == \ + f'hash:{scrubber.placeholder_hash}' + + assert scrubber.user_name is None + + +def test_scrub_text_complex_string(scrubber): + source = 'this is a string that have been sent from ' \ + '192.168.1.1(3030303030303030303030303030303030303030) ' \ + 'located at usr/someuser/path at ' \ + 'someuser machine(someuserany)' + + actual = scrubber.scrub_text(source) + + assert actual == f'this is a string that have been sent from ' \ + f'{scrubber.placeholder_ip}({scrubber.placeholder_hash}) ' \ + f'located at usr/{scrubber.placeholder_user}/path at ' \ + f'{scrubber.placeholder_user} machine(someuserany)' + + assert scrubber.user_name == 'someuser' + assert scrubber.scrub_text('someuser') == scrubber.placeholder_user + + +def test_scrub_simple_event(scrubber): + assert scrubber.scrub_event(None) is None + assert scrubber.scrub_event({}) == {} + assert scrubber.scrub_event({'some': 'field'}) == {'some': 'field'} + + +def test_scrub_event(scrubber): + event = { + CONTEXTS: { + REPORTER: { + OS_ENVIRON: { + 'PATH': '/users/username/apps' + }, + STACKTRACE: ['Traceback (most recent call last):', + 'File "/Users/username/Tribler/tribler/src/tribler-gui/tribler_gui/"'], + SYSINFO: {'sys.path': ['/Users/username/Tribler/', + '/Users/username/', + '.']}, + } + }, + EXTRA: { + SYS_ARGV: ['/Users/username/Tribler'] + }, + LOGENTRY: { + 'message': 'Exception with username', + 'params': ['Traceback File: /Users/username/Tribler/'] + }, + BREADCRUMBS: { + 'values': [ + {'type': 'log', 'message': 'Traceback File: /Users/username/Tribler/'}, + {'type': 'log', 'message': 'IP: 192.168.1.1'} + ] + }, + + } + + assert scrubber.scrub_event(event) == { + CONTEXTS: { + REPORTER: { + OS_ENVIRON: { + 'PATH': f'/users/{scrubber.placeholder_user}/apps' + }, + STACKTRACE: ['Traceback (most recent call last):', + f'File "/Users/{scrubber.placeholder_user}/Tribler/tribler/src/tribler-gui/tribler_gui/"'], + SYSINFO: {'sys.path': [f'/Users/{scrubber.placeholder_user}/Tribler/', + f'/Users/{scrubber.placeholder_user}/', + '.']}, + }, + }, + LOGENTRY: { + 'message': f'Exception with {scrubber.placeholder_user}', + 'params': [f'Traceback File: /Users/{scrubber.placeholder_user}/Tribler/'] + }, + EXTRA: { + SYS_ARGV: [f'/Users/{scrubber.placeholder_user}/Tribler'] + }, + BREADCRUMBS: { + 'values': [ + {'type': 'log', + 'message': f'Traceback File: /Users/{scrubber.placeholder_user}/Tribler/'}, + {'type': 'log', + 'message': f'IP: {scrubber.placeholder_ip}'} + ] + }, + } + + +def test_entities_recursively(scrubber): + # positive + assert scrubber.scrub_entity_recursively(None) is None + assert scrubber.scrub_entity_recursively({}) == {} + assert scrubber.scrub_entity_recursively([]) == [] + assert scrubber.scrub_entity_recursively('') == '' + assert scrubber.scrub_entity_recursively(42) == 42 + + event = {'some': {'value': [{'path': '/Users/username/Tribler'}]}} + assert scrubber.scrub_entity_recursively(event) == { + 'some': {'value': [{'path': f'/Users/{scrubber.placeholder_user}/Tribler'}]}} + # stop on depth + + assert scrubber.scrub_entity_recursively(event) != event + assert scrubber.scrub_entity_recursively(event, depth=2) == event + + +def test_scrub_unnecessary_fields(scrubber): + # default + assert scrubber.scrub_event({'default': 'field'}) == {'default': 'field'} + assert scrubber.scrub_event({'modules': {}}) == {} + + # custom + custom_scrubber = SentryScrubber() + custom_scrubber.event_fields_to_cut = ['new', 'default'] + assert custom_scrubber.scrub_event( + { + 'default': 'event', + 'new': 'field', + 'modules': {} + + }) == { + 'modules': {} + } + + +def test_scrub_text_none(scrubber): + assert scrubber.scrub_text(None) is None + + +def test_scrub_text_some(scrubber): + assert scrubber.scrub_text('some text') == 'some text' + assert scrubber.user_name is None + + +def test_scrub_dict(scrubber): + assert scrubber.scrub_entity_recursively(None) is None + assert scrubber.scrub_entity_recursively({}) == {} + + assert scrubber.scrub_entity_recursively({'PATH': '/home/username/some/', + 'USER': 'username'}) \ + == {'PATH': f'/home/{scrubber.placeholder_user}/some/', + 'USER': scrubber.placeholder_user} + assert scrubber.user_name == 'username' + + +def test_scrub_list(scrubber): + assert scrubber.scrub_entity_recursively(None) is None + assert scrubber.scrub_entity_recursively([]) == [] + + assert scrubber.scrub_entity_recursively(['/home/username/some/']) == \ + [f'/home/{scrubber.placeholder_user}/some/'] + assert scrubber.user_name == 'username' diff --git a/src/tribler-common/tribler_common/sentry_reporter/tests/test_sentry_tools.py b/src/tribler-common/tribler_common/sentry_reporter/tests/test_sentry_tools.py new file mode 100644 index 00000000000..a4dd3159696 --- /dev/null +++ b/src/tribler-common/tribler_common/sentry_reporter/tests/test_sentry_tools.py @@ -0,0 +1,88 @@ +from tribler_common.sentry_reporter.sentry_tools import ( + delete_item, + get_first_item, + get_last_item, + get_value, + modify_value, + parse_os_environ, + parse_stacktrace, +) + + +def test_first(): + assert get_first_item(None, '') == '' + assert get_first_item([], '') == '' + assert get_first_item(['some'], '') == 'some' + assert get_first_item(['some', 'value'], '') == 'some' + + assert get_first_item((), '') == '' + assert get_first_item(('some', 'value'), '') == 'some' + + assert get_first_item(None, None) is None + + +def test_last(): + assert get_last_item(None, '') == '' + assert get_last_item([], '') == '' + assert get_last_item(['some'], '') == 'some' + assert get_last_item(['some', 'value'], '') == 'value' + + assert get_last_item((), '') == '' + assert get_last_item(('some', 'value'), '') == 'value' + + assert get_last_item(None, None) is None + + +def test_delete(): + assert delete_item({}, None) == {} + + assert delete_item({'key': 'value'}, None) == {'key': 'value'} + assert delete_item({'key': 'value'}, 'missed_key') == {'key': 'value'} + assert delete_item({'key': 'value'}, 'key') == {} + + +def test_parse_os_environ(): + # simple tests + assert parse_os_environ(None) == {} + assert parse_os_environ([]) == {} + assert parse_os_environ(['KEY:value']) == {'KEY': 'value'} + + assert parse_os_environ(['KEY:value', 'KEY1:value1', 'KEY2:value2']) == { + 'KEY': 'value', + 'KEY1': 'value1', + 'KEY2': 'value2', + } + + # test multiply `:` + assert parse_os_environ(['KEY:value:and:some']) == {'KEY': 'value:and:some'} + + # test no `:` + assert parse_os_environ(['KEY:value', 'key']) == {'KEY': 'value'} + + +def test_parse_stacktrace(): + assert parse_stacktrace(None) == [] + assert parse_stacktrace('') == [] + assert parse_stacktrace('\n') == [] + assert parse_stacktrace('\n\n') == [] + assert parse_stacktrace('some\n\nvalue') == ['some', 'value'] + + +def test_modify(): + assert modify_value(None, None, None) is None + assert modify_value({}, None, None) == {} + assert modify_value({}, '', None) == {} + + assert modify_value({}, 'key', lambda value: '') == {} + assert modify_value({'a': 'b'}, 'key', lambda value: '') == {'a': 'b'} + assert modify_value({'a': 'b', 'key': 'value'}, 'key', lambda value: '') == {'a': 'b', 'key': ''} + + +def test_safe_get(): + assert get_value(None, None, None) is None + assert get_value(None, None, {}) == {} + + assert get_value(None, 'key', {}) == {} + + assert get_value({'key': 'value'}, 'key', {}) == 'value' + assert get_value({'key': 'value'}, 'key1', {}) == {} diff --git a/src/tribler-core/tribler_core/restapi/events_endpoint.py b/src/tribler-core/tribler_core/restapi/events_endpoint.py index e6623ad7437..c11fa8fba87 100644 --- a/src/tribler-core/tribler_core/restapi/events_endpoint.py +++ b/src/tribler-core/tribler_core/restapi/events_endpoint.py @@ -113,9 +113,14 @@ async def write_data(self, message): for request in self.events_responses: await request.write(message_bytes) - # An exception has occurred in Tribler. The event includes a readable string of the error. - def on_tribler_exception(self, exception_text): - self.write_data({"type": NTFY.TRIBLER_EXCEPTION.value, "event": {"text": exception_text}}) + # An exception has occurred in Tribler. The event includes a readable + # string of the error and a Sentry event. + def on_tribler_exception(self, exception_text, sentry_event): + self.write_data({ + "type": NTFY.TRIBLER_EXCEPTION.value, + "event": {"text": exception_text}, + "sentry_event": sentry_event + }) @docs( tags=["General"], diff --git a/src/tribler-core/tribler_core/restapi/state_endpoint.py b/src/tribler-core/tribler_core/restapi/state_endpoint.py index fd010c48f73..952c72fadf4 100644 --- a/src/tribler-core/tribler_core/restapi/state_endpoint.py +++ b/src/tribler-core/tribler_core/restapi/state_endpoint.py @@ -20,6 +20,7 @@ def __init__(self, session): super(StateEndpoint, self).__init__(session) self.tribler_state = STATE_STARTING self.last_exception = None + self.sentry_event = None self.session.notifier.add_observer(NTFY.UPGRADER_STARTED, self.on_tribler_upgrade_started) self.session.notifier.add_observer(NTFY.UPGRADER_DONE, self.on_tribler_upgrade_finished) @@ -37,9 +38,10 @@ def on_tribler_upgrade_finished(self, *_): def on_tribler_started(self, *_): self.tribler_state = STATE_STARTED - def on_tribler_exception(self, exception_text): + def on_tribler_exception(self, exception_text, sentry_event): self.tribler_state = STATE_EXCEPTION self.last_exception = exception_text + self.sentry_event = sentry_event @docs( tags=["General"], diff --git a/src/tribler-core/tribler_core/restapi/tests/test_events_endpoint.py b/src/tribler-core/tribler_core/restapi/tests/test_events_endpoint.py index b1626c34e43..94678916c22 100644 --- a/src/tribler-core/tribler_core/restapi/tests/test_events_endpoint.py +++ b/src/tribler-core/tribler_core/restapi/tests/test_events_endpoint.py @@ -49,13 +49,13 @@ async def test_events(enable_api, session): testdata = { NTFY.CHANNEL_ENTITY_UPDATED: {"state": "Complete"}, - NTFY.UPGRADER_TICK: ("bla", ), + NTFY.UPGRADER_TICK: ("bla",), NTFY.UPGRADER_DONE: None, - NTFY.WATCH_FOLDER_CORRUPT_FILE: ("foo", ), + NTFY.WATCH_FOLDER_CORRUPT_FILE: ("foo",), NTFY.TRIBLER_NEW_VERSION: ("123",), NTFY.CHANNEL_DISCOVERED: {"result": "bla"}, NTFY.TORRENT_FINISHED: (b'a' * 10, None, False), - NTFY.LOW_SPACE: ("", ), + NTFY.LOW_SPACE: ("",), NTFY.TUNNEL_REMOVE: (Circuit(1234, None), 'test'), NTFY.REMOTE_QUERY_RESULTS: {"query": "test"}, } @@ -66,7 +66,7 @@ async def test_events(enable_api, session): session.notifier.notify(subject, *data) else: session.notifier.notify(subject) - session.api_manager.root_endpoint.endpoints['/events'].on_tribler_exception("hi") + session.api_manager.root_endpoint.endpoints['/events'].on_tribler_exception("hi", None) await events_future event_socket_task.cancel() diff --git a/src/tribler-core/tribler_core/session.py b/src/tribler-core/tribler_core/session.py index 7d30f0d40b2..54762a92a19 100644 --- a/src/tribler-core/tribler_core/session.py +++ b/src/tribler-core/tribler_core/session.py @@ -3,7 +3,6 @@ Author(s): Arno Bakker, Niels Zeilmaker, Vadim Bulavintsev """ - import errno import logging import os @@ -21,6 +20,7 @@ from ipv8_service import IPv8 +from tribler_common.sentry_reporter.sentry_reporter import AllowSentryReports, SentryReporter from tribler_common.simpledefs import ( NTFY, STATEDIR_CHANNELS_DIR, @@ -82,7 +82,7 @@ class Session(TaskManager): """ __single = None - def __init__(self, config, core_test_mode = False): + def __init__(self, config, core_test_mode=False): """ A Session object is created Only a single session instance can exist at a time in a process. @@ -162,7 +162,7 @@ async def start_bootstrap_download(self): self.bootstrap.start_by_infohash(self.dlmgr.start_download, self.config.get_bootstrap_infohash()) await self.bootstrap.download.future_finished # Temporarily disabling SQL import for experimental release - #await get_event_loop().run_in_executor(None, self.import_bootstrap_file) + # await get_event_loop().run_in_executor(None, self.import_bootstrap_file) self.bootstrap.bootstrap_finished = True def create_state_directory_structure(self): @@ -221,7 +221,6 @@ def unhandled_error_observer(self, loop, context): It broadcasts the tribler_exception event. """ exception = context.get('exception') - ignored_message = None try: ignored_message = IGNORED_ERRORS.get( @@ -249,11 +248,20 @@ def unhandled_error_observer(self, loop, context): text_long = text_long + "\n--LONG TEXT--\n" + buffer.getvalue() text_long = text_long + "\n--CONTEXT--\n" + str(context) - self._logger.error("Unhandled exception occurred! %s", text_long) + description = 'session.unhandled_error_observer()' + with AllowSentryReports(value=False, description=description): + self._logger.error("Unhandled exception occurred! %s", text_long) - if self.api_manager and len(text_long) > 0: - self.api_manager.get_endpoint('events').on_tribler_exception(text_long) - self.api_manager.get_endpoint('state').on_tribler_exception(text_long) + sentry_event = SentryReporter.last_event + + if not self.api_manager: + return + + events = self.api_manager.get_endpoint('events') + events.on_tribler_exception(text_long, sentry_event) + + state = self.api_manager.get_endpoint('state') + state.on_tribler_exception(text_long, sentry_event) def get_tribler_statistics(self): """Return a dictionary with general Tribler statistics.""" @@ -270,11 +278,11 @@ async def start(self): :param config: a TriblerConfig object """ - self._logger.info("Session is using state directory: %s", self.config.get_state_dir()) self.get_ports_in_config() self.create_state_directory_structure() self.init_keypair() + SentryReporter.set_user(self.trustchain_keypair.key.pk) # Start the REST API before the upgrader since we want to send interesting upgrader events over the socket if self.config.get_api_http_enabled() or self.config.get_api_https_enabled(): @@ -382,7 +390,7 @@ async def start(self): # GigaChannel Manager should be started *after* resuming the downloads, # because it depends on the states of torrent downloads - if self.config.get_chant_enabled() and self.config.get_chant_manager_enabled()\ + if self.config.get_chant_enabled() and self.config.get_chant_manager_enabled() \ and self.config.get_libtorrent_enabled: self.gigachannel_manager = GigaChannelManager(self) if not self.core_test_mode: diff --git a/src/tribler-core/tribler_core/version.py b/src/tribler-core/tribler_core/version.py index 1ebda0abb5e..471bc30dd10 100644 --- a/src/tribler-core/tribler_core/version.py +++ b/src/tribler-core/tribler_core/version.py @@ -1,3 +1,4 @@ version_id = "7.5.0-GIT" build_date = "Mon Jan 01 00:00:01 1970" commit_id = "none" +sentry_url = "" diff --git a/src/tribler-gui/tribler_gui/dialogs/feedbackdialog.py b/src/tribler-gui/tribler_gui/dialogs/feedbackdialog.py index 813bcade064..7024c9c6898 100644 --- a/src/tribler-gui/tribler_gui/dialogs/feedbackdialog.py +++ b/src/tribler-gui/tribler_gui/dialogs/feedbackdialog.py @@ -3,22 +3,23 @@ import platform import sys import time +from collections import defaultdict from PyQt5 import uic from PyQt5.QtWidgets import QAction, QApplication, QDialog, QMessageBox, QTreeWidgetItem +from tribler_common.sentry_reporter.sentry_reporter import SentryReporter + from tribler_gui.event_request_manager import received_events from tribler_gui.tribler_action_menu import TriblerActionMenu from tribler_gui.tribler_request_manager import ( - TriblerNetworkRequest, - performed_requests as tribler_performed_requests, - tribler_urlencode, -) + TriblerNetworkRequest, performed_requests as tribler_performed_requests, tribler_urlencode, ) from tribler_gui.utilities import get_ui_file_path class FeedbackDialog(QDialog): - def __init__(self, parent, exception_text, tribler_version, start_time): + def __init__(self, parent, exception_text, tribler_version, start_time, # pylint: disable=R0914 + backend_sentry_event=None): QDialog.__init__(self, parent) uic.loadUi(get_ui_file_path('feedback_dialog.ui'), self) @@ -26,6 +27,7 @@ def __init__(self, parent, exception_text, tribler_version, start_time): self.setWindowTitle("Unexpected error") self.selected_item_index = 0 self.tribler_version = tribler_version + self.backend_sentry_event = backend_sentry_event # Qt 5.2 does not have the setPlaceholderText property if hasattr(self.comments_text_edit, "setPlaceholderText"): @@ -124,9 +126,14 @@ def on_send_clicked(self): endpoint = 'http://reporter.tribler.org/report' sys_info = "" + sys_info_dict = defaultdict(lambda: []) for ind in range(self.env_variables_list.topLevelItemCount()): item = self.env_variables_list.topLevelItem(ind) - sys_info += "%s\t%s\n" % (item.text(0), item.text(1)) + key = item.text(0) + value = item.text(1) + + sys_info += "%s\t%s\n" % (key, value) + sys_info_dict[key].append(value) comments = self.comments_text_edit.toPlainText() if len(comments) == 0: @@ -143,6 +150,10 @@ def on_send_clicked(self): "stack": stack, } + # if no backend_sentry_event, then try to restore GUI exception + sentry_event = self.backend_sentry_event or SentryReporter.last_event + SentryReporter.send(sentry_event, post_data, sys_info_dict) + TriblerNetworkRequest(endpoint, self.on_report_sent, raw_data=tribler_urlencode(post_data), method='POST') def closeEvent(self, close_event): diff --git a/src/tribler-gui/tribler_gui/event_request_manager.py b/src/tribler-gui/tribler_gui/event_request_manager.py index f52749041b2..cf151a25ef4 100644 --- a/src/tribler-gui/tribler_gui/event_request_manager.py +++ b/src/tribler-gui/tribler_gui/event_request_manager.py @@ -99,7 +99,9 @@ def on_read_data(self): else: reaction() elif event_type == NTFY.TRIBLER_EXCEPTION.value: - raise RuntimeError(json_dict["event"]["text"]) + text = json_dict["event"]["text"] + sentry_event = {'sentry_event': json_dict['sentry_event']} + raise RuntimeError(text, sentry_event) self.current_event_string = "" def on_finished(self): diff --git a/src/tribler-gui/tribler_gui/tribler_window.py b/src/tribler-gui/tribler_gui/tribler_window.py index f0d943a23bf..809f52a7a1a 100644 --- a/src/tribler-gui/tribler_gui/tribler_window.py +++ b/src/tribler-gui/tribler_gui/tribler_window.py @@ -38,6 +38,8 @@ QTreeWidget, ) +from tribler_common.sentry_reporter.sentry_reporter import AllowSentryReports + from tribler_core.modules.process_checker import ProcessChecker from tribler_core.utilities.unicode import hexlify from tribler_core.version import version_id @@ -103,11 +105,20 @@ def on_exception(self, *exc_info): return self.exception_handler_called = True - + info_type, info_error, _ = exc_info exception_text = "".join(traceback.format_exception(*exc_info)) - logging.error(exception_text) - self.tribler_crashed.emit(exception_text) + backend_sentry_event = None + for arg in info_error.args: + key = 'sentry_event' + if isinstance(arg, dict) and key in arg: + backend_sentry_event = arg[key] + break + + with AllowSentryReports(value=False, description='TriblerWindow.on_exception()'): + logging.error(exception_text) + + self.tribler_crashed.emit(exception_text) self.delete_tray_icon() # Stop the download loop @@ -124,10 +135,10 @@ def on_exception(self, *exc_info): if self.debug_window: self.debug_window.setHidden(True) - if exc_info[0] is CoreConnectTimeoutError: + if info_type is CoreConnectTimeoutError: exception_text = exception_text + self.core_manager.core_traceback - dialog = FeedbackDialog(self, exception_text, self.tribler_version, self.start_time) + dialog = FeedbackDialog(self, exception_text, self.tribler_version, self.start_time, backend_sentry_event) dialog.show() def __init__(self, core_args=None, core_env=None, api_port=None, api_key=None): @@ -170,7 +181,6 @@ def __init__(self, core_args=None, core_env=None, api_port=None, api_key=None): self.add_torrent_url_dialog_active = False sys.excepthook = self.on_exception - uic.loadUi(get_ui_file_path('mainwindow.ui'), self) TriblerRequestManager.window = self self.tribler_status_bar.hide()