From 7c0213ef4799937d14bb436959f4b27c3927ec41 Mon Sep 17 00:00:00 2001 From: Alexander Kozlovsky Date: Fri, 24 Dec 2021 18:40:54 +0100 Subject: [PATCH 01/16] Fix sys.path in tribler.spec --- tribler.spec | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tribler.spec b/tribler.spec index 963a013ebdf..9d4a6fe31fd 100644 --- a/tribler.spec +++ b/tribler.spec @@ -13,13 +13,7 @@ from PyInstaller.utils.hooks import collect_data_files, collect_submodules root_dir = os.path.abspath(os.path.dirname(__name__)) src_dir = os.path.join(root_dir, "src") - -tribler_components = [ - os.path.join(src_dir, "tribler"), -] - -for component in tribler_components: - sys.path.append(str(component)) +sys.path.append(src_dir) from tribler.core.version import version_id version_str = version_id.split('-')[0] From a5e97958aaac449aa5466cbfbd689fcf50698be7 Mon Sep 17 00:00:00 2001 From: Alexander Kozlovsky Date: Thu, 12 May 2022 11:30:11 +0200 Subject: [PATCH 02/16] Set debug log level for PyInstaller on all systems --- build/debian/makedist_debian.sh | 2 +- build/mac/makedist_macos.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build/debian/makedist_debian.sh b/build/debian/makedist_debian.sh index 2ea71e7a6af..8e4a700bce8 100755 --- a/build/debian/makedist_debian.sh +++ b/build/debian/makedist_debian.sh @@ -22,7 +22,7 @@ python3 build/update_version_from_git.py # ----- Install pip dependencies before the build python3 -m pip install --upgrade -r requirements.txt -python3 -m PyInstaller tribler.spec +python3 -m PyInstaller tribler.spec --log-level=DEBUG cp -r dist/tribler build/debian/tribler/usr/share/tribler diff --git a/build/mac/makedist_macos.sh b/build/mac/makedist_macos.sh index 343ae882504..a3f0fbf0e04 100755 --- a/build/mac/makedist_macos.sh +++ b/build/mac/makedist_macos.sh @@ -17,7 +17,7 @@ export RESOURCES=build/mac/resources python3 -m pip install --upgrade -r requirements.txt PI=pyinstaller -$PI tribler.spec +$PI tribler.spec --log-level=DEBUG mkdir -p dist/installdir mv dist/$APPNAME.app dist/installdir From 7808ec094fc1042697afa75d8ce7902472c17dc0 Mon Sep 17 00:00:00 2001 From: Alexander Kozlovsky Date: Wed, 13 Apr 2022 00:23:23 +0200 Subject: [PATCH 03/16] Add ipv8 to hidden imports --- tribler.spec | 1 + 1 file changed, 1 insertion(+) diff --git a/tribler.spec b/tribler.spec index 9d4a6fe31fd..e6f9be727b6 100644 --- a/tribler.spec +++ b/tribler.spec @@ -84,6 +84,7 @@ hiddenimports = [ 'csv', 'dataclasses', # https://github.com/pyinstaller/pyinstaller/issues/5432 'ecdsa', + 'ipv8', 'PIL', 'pkg_resources', # 'pkg_resources.py2_warn', # Workaround PyInstaller & SetupTools, https://github.com/pypa/setuptools/issues/1963 'pyaes', From 5b482d6668ac6ebdf59340e919b7b5dfbeba32c4 Mon Sep 17 00:00:00 2001 From: Alexander Kozlovsky Date: Wed, 13 Apr 2022 08:17:01 +0200 Subject: [PATCH 04/16] Print detailed error description in case of logger config initialization error --- src/tribler/core/logger/logger.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/tribler/core/logger/logger.py b/src/tribler/core/logger/logger.py index a6f66cabe95..042ed0cff20 100644 --- a/src/tribler/core/logger/logger.py +++ b/src/tribler/core/logger/logger.py @@ -63,5 +63,14 @@ def setup_logging(app_mode, log_dir: Path, config_path: Path): logging.config.dictConfig(config) logger.info(f'Config loaded for app_mode={app_mode}') except Exception as e: # pylint: disable=broad-except - print('Error in loading logger config. Using default configs. Error:', e, file=sys.stderr) + error_description = format_error_description(e) + print('Error in loading logger config. Using default configs. ', error_description, file=sys.stderr) logging.basicConfig(level=logging.INFO, stream=sys.stdout) + + +def format_error_description(e: Exception): + result = f'{e.__class__.__name__}: {e}' + cause = e.__cause__ + if cause: + result += f'. Cause: {cause.__class__.__name__}: {cause}' + return result From 4238b92ce60a57deb427a60913dc628e564f4b76 Mon Sep 17 00:00:00 2001 From: Alexander Kozlovsky Date: Wed, 13 Apr 2022 08:30:46 +0200 Subject: [PATCH 05/16] Add tribler.core.logger.logger_streams to hidden imports --- tribler.spec | 1 + 1 file changed, 1 insertion(+) diff --git a/tribler.spec b/tribler.spec index e6f9be727b6..9b1cd071d25 100644 --- a/tribler.spec +++ b/tribler.spec @@ -97,6 +97,7 @@ hiddenimports = [ 'requests', 'scrypt', '_scrypt', 'sqlalchemy', 'sqlalchemy.ext.baked', 'sqlalchemy.ext.declarative', + 'tribler.core.logger.logger_streams', 'typing_extensions', ] + widget_files + pony_deps + get_sentry_hooks() From 8497ffad7339ba6450360e0c33bad70948f87b37 Mon Sep 17 00:00:00 2001 From: Alexander Kozlovsky Date: Wed, 13 Apr 2022 10:15:41 +0200 Subject: [PATCH 06/16] Suppress exceptions in StreamWrapper.flush() to avoid "Invalid argument" error on some systems --- src/tribler/core/logger/logger_streams.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/tribler/core/logger/logger_streams.py b/src/tribler/core/logger/logger_streams.py index 1ca37ab6df6..0e7008964db 100644 --- a/src/tribler/core/logger/logger_streams.py +++ b/src/tribler/core/logger/logger_streams.py @@ -11,7 +11,10 @@ def __init__(self, stream: TextIO): self.stream = stream def flush(self): - self.stream.flush() + try: + self.stream.flush() + except: + pass def write(self, s: str): try: @@ -22,7 +25,10 @@ def write(self, s: str): self.stream.write(s2) def close(self): - self.stream.close() + try: + self.stream.close() + except: + pass stdout_wrapper = StreamWrapper(sys.stdout) # specified in logger.yaml for `console` handler From 6c1c671d227c829ef86c3a914c3b9de6d239809b Mon Sep 17 00:00:00 2001 From: Alexander Kozlovsky Date: Mon, 2 May 2022 17:36:06 +0200 Subject: [PATCH 07/16] Install PyInstaller into venv with pip to fix build on win32 --- build/win/makedist_win.bat | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/build/win/makedist_win.bat b/build/win/makedist_win.bat index d9d59dffa7b..93b4e9d5707 100644 --- a/build/win/makedist_win.bat +++ b/build/win/makedist_win.bat @@ -16,15 +16,6 @@ REM Arno: Add . to find our core SET PYTHONPATH=.;%PYTHONHOME% ECHO PYTHONPATH SET TO %PYTHONPATH% -REM ----- Check for PyInstaller - -IF NOT EXIST %PYTHONHOME%\Scripts\pyinstaller.exe ( - ECHO . - ECHO Could not locate pyinstaller in %PYTHONHOME%\Scripts. - ECHO Please modify this script or install PyInstaller [www.pyinstaller.org] - EXIT /b -) - REM ----- Check for NSIS installer SET NSIS="C:\Program Files\NSIS\makensis.exe" @@ -48,6 +39,9 @@ python3 -m pip install --upgrade -r requirements.txt REM Arno: When adding files here, make sure tribler.nsi actually REM packs them in the installer .EXE +REM ----- Install PyInstaller +python3 -m pip install --upgrade PyInstaller + ECHO Install pip dependencies for correct py-installer's work python3 -m pip install --upgrade -r build\win\requirements.txt From c3ce22d1d83e2c5deb3387f5a095265037b5ca10 Mon Sep 17 00:00:00 2001 From: Alexander Kozlovsky Date: Tue, 3 May 2022 06:45:04 +0200 Subject: [PATCH 08/16] Upgrade pip before installing requirements to fix mysterious problems with hidden imports in PyInstaller in Windows build --- build/debian/makedist_debian.sh | 3 +++ build/mac/makedist_macos.sh | 4 ++++ build/win/makedist_win.bat | 3 +++ 3 files changed, 10 insertions(+) diff --git a/build/debian/makedist_debian.sh b/build/debian/makedist_debian.sh index 8e4a700bce8..60dd59515e3 100755 --- a/build/debian/makedist_debian.sh +++ b/build/debian/makedist_debian.sh @@ -19,6 +19,9 @@ rm -rf build/debian/tribler/usr/share/tribler python3 build/update_version_from_git.py +# ----- Upgrade pip to fix potential PyInstaller problems with module discovering +python3 -m pip install --upgrade pip + # ----- Install pip dependencies before the build python3 -m pip install --upgrade -r requirements.txt diff --git a/build/mac/makedist_macos.sh b/build/mac/makedist_macos.sh index a3f0fbf0e04..f9a77e04ceb 100755 --- a/build/mac/makedist_macos.sh +++ b/build/mac/makedist_macos.sh @@ -13,6 +13,10 @@ export RESOURCES=build/mac/resources /bin/rm -rf dist # ----- Build + +# ----- Upgrade pip to fix potential PyInstaller problems with module discovering +python3 -m pip install --upgrade pip + # ----- Install pip dependencies before the build python3 -m pip install --upgrade -r requirements.txt diff --git a/build/win/makedist_win.bat b/build/win/makedist_win.bat index 93b4e9d5707..f9af06d4be2 100644 --- a/build/win/makedist_win.bat +++ b/build/win/makedist_win.bat @@ -33,6 +33,9 @@ call build\win\clean.bat REM ----- Build +REM ----- Upgrade pip to fix potential PyInstaller problems with module discovering +python3 -m pip install --upgrade pip + REM ----- Install pip dependencies before the build python3 -m pip install --upgrade -r requirements.txt From 2cfb045f8037dde75ac85125db34c80865bd8c3c Mon Sep 17 00:00:00 2001 From: Alexander Kozlovsky Date: Thu, 12 May 2022 14:29:47 +0200 Subject: [PATCH 09/16] Build from venv --- build/debian/makedist_debian.sh | 9 ++++++--- build/mac/makedist_macos.sh | 9 +++++---- build/win/makedist_win.bat | 13 ++++++------- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/build/debian/makedist_debian.sh b/build/debian/makedist_debian.sh index 60dd59515e3..fec31a0346b 100755 --- a/build/debian/makedist_debian.sh +++ b/build/debian/makedist_debian.sh @@ -19,12 +19,15 @@ rm -rf build/debian/tribler/usr/share/tribler python3 build/update_version_from_git.py -# ----- Upgrade pip to fix potential PyInstaller problems with module discovering -python3 -m pip install --upgrade pip +# ----- Prepare venv & install dependencies before the build -# ----- Install pip dependencies before the build +python3 -m venv build-env +. ./build-env/bin/activate +python3 -m pip install --upgrade pip python3 -m pip install --upgrade -r requirements.txt +# ----- Build + python3 -m PyInstaller tribler.spec --log-level=DEBUG cp -r dist/tribler build/debian/tribler/usr/share/tribler diff --git a/build/mac/makedist_macos.sh b/build/mac/makedist_macos.sh index f9a77e04ceb..cff24fca150 100755 --- a/build/mac/makedist_macos.sh +++ b/build/mac/makedist_macos.sh @@ -12,14 +12,15 @@ export RESOURCES=build/mac/resources # ----- Clean up /bin/rm -rf dist -# ----- Build +# ----- Prepare venv & install dependencies before the build -# ----- Upgrade pip to fix potential PyInstaller problems with module discovering +python3 -m venv build-env +. ./build-env/bin/activate python3 -m pip install --upgrade pip - -# ----- Install pip dependencies before the build python3 -m pip install --upgrade -r requirements.txt +# ----- Build + PI=pyinstaller $PI tribler.spec --log-level=DEBUG diff --git a/build/win/makedist_win.bat b/build/win/makedist_win.bat index f9af06d4be2..6272309e017 100644 --- a/build/win/makedist_win.bat +++ b/build/win/makedist_win.bat @@ -31,20 +31,19 @@ REM ----- Clean up call build\win\clean.bat -REM ----- Build +REM ----- Prepare venv & install dependencies before the build -REM ----- Upgrade pip to fix potential PyInstaller problems with module discovering +python3 -m venv build-env +./build-env/Scripts/activate.bat python3 -m pip install --upgrade pip - -REM ----- Install pip dependencies before the build python3 -m pip install --upgrade -r requirements.txt +python3 -m pip install --upgrade PyInstaller + +REM ----- Build REM Arno: When adding files here, make sure tribler.nsi actually REM packs them in the installer .EXE -REM ----- Install PyInstaller -python3 -m pip install --upgrade PyInstaller - ECHO Install pip dependencies for correct py-installer's work python3 -m pip install --upgrade -r build\win\requirements.txt From a469fb1fc570eeb44260aeb2fd22cbc539d0e66b Mon Sep 17 00:00:00 2001 From: Alexander Kozlovsky Date: Wed, 11 May 2022 13:12:03 +0200 Subject: [PATCH 10/16] Update requirements to correspond with main branch --- requirements-core.txt | 5 +++-- requirements-test.txt | 20 ++++++++++---------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/requirements-core.txt b/requirements-core.txt index ea35c5499d4..64b882f4ea7 100644 --- a/requirements-core.txt +++ b/requirements-core.txt @@ -14,7 +14,7 @@ networkx==2.6.3 pony==0.7.14 psutil==5.8.0 pyasn1==0.4.8 -pydantic==1.8.2 +pydantic==1.9.0 PyOpenSSL==21.0.0 pyyaml==6.0 sentry-sdk==1.5.0 @@ -23,4 +23,5 @@ yappi==1.3.3 yarl==1.7.2 # keep this dependency higher than 1.6.3. See: https://github.com/aio-libs/yarl/issues/517 Faker==9.8.2 sentry-sdk==1.5.0 -pyipv8==2.8.0 \ No newline at end of file +pyipv8==2.8.0 +libtorrent==1.2.15 diff --git a/requirements-test.txt b/requirements-test.txt index 6402c6ffb51..eed40540517 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,14 +1,14 @@ -r requirements.txt -pytest==6.2.5 -pytest-aiohttp==0.3.0 -pytest-asyncio==0.16.0 -pytest-cov==3.0.0 -pytest-mock==3.6.1 -pytest-randomly==3.10.2 -pytest-timeout==2.0.1 -pytest-xdist==2.4.0 +pytest==7.1.2 +pytest-aiohttp==1.0.4 +pytest-asyncio==0.18.3 +pytest-mock==3.7.0 +pytest-randomly==3.11.0 +pytest-timeout==2.1.0 pytest-freezegun==0.4.2 -freezegun==1.1.0 +freezegun==1.2.1 +coverage==6.3.2 +looptime==0.2 -asynctest==0.13.0 +asynctest==0.13.0 # this library has to be installed to properly work with ipv8 TestBase. \ No newline at end of file From 8608d5b291aa32e222ef544cbd2a8e3901bcabdb Mon Sep 17 00:00:00 2001 From: Alexander Kozlovsky Date: Thu, 12 May 2022 10:45:29 +0200 Subject: [PATCH 11/16] Remove duplicate requirements --- requirements-core.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/requirements-core.txt b/requirements-core.txt index 64b882f4ea7..60f4293437e 100644 --- a/requirements-core.txt +++ b/requirements-core.txt @@ -21,7 +21,5 @@ sentry-sdk==1.5.0 service-identity==21.1.0 yappi==1.3.3 yarl==1.7.2 # keep this dependency higher than 1.6.3. See: https://github.com/aio-libs/yarl/issues/517 -Faker==9.8.2 -sentry-sdk==1.5.0 pyipv8==2.8.0 libtorrent==1.2.15 From 058ae35463e0c2b9e95fafd4191bb130068e0156 Mon Sep 17 00:00:00 2001 From: Alexander Kozlovsky Date: Thu, 12 May 2022 10:45:04 +0200 Subject: [PATCH 12/16] Update pony version in requirements --- requirements-core.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-core.txt b/requirements-core.txt index 60f4293437e..2e08099766c 100644 --- a/requirements-core.txt +++ b/requirements-core.txt @@ -11,7 +11,7 @@ lz4==3.1.3 marshmallow==3.14.1 netifaces==0.11.0 networkx==2.6.3 -pony==0.7.14 +pony==0.7.16 psutil==5.8.0 pyasn1==0.4.8 pydantic==1.9.0 From d5c134ffbd18a8d588a8e17ac82dd3e234e8e397 Mon Sep 17 00:00:00 2001 From: Alexander Kozlovsky Date: Thu, 12 May 2022 14:28:30 +0200 Subject: [PATCH 13/16] Update windows-specific requirements --- build/win/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/win/requirements.txt b/build/win/requirements.txt index 3dd795b4eb5..0e568918991 100644 --- a/build/win/requirements.txt +++ b/build/win/requirements.txt @@ -1,4 +1,4 @@ --no-binary :all: -typing_extensions==3.10.0.2 -pydantic==1.8.2 \ No newline at end of file +typing_extensions==4.2.0 +pydantic==1.9.0 From 7396e226cf834da8b2c4532f80544ed8a90683a0 Mon Sep 17 00:00:00 2001 From: Alexander Kozlovsky Date: Mon, 2 May 2022 08:56:10 +0200 Subject: [PATCH 14/16] Reduce the scope of try/except handling for binascii.Error while executing a remote code --- src/tribler/gui/code_executor.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/tribler/gui/code_executor.py b/src/tribler/gui/code_executor.py index eee80d8e50e..0b961641e67 100644 --- a/src/tribler/gui/code_executor.py +++ b/src/tribler/gui/code_executor.py @@ -84,10 +84,12 @@ def _on_socket_read_ready(self): try: code = b64decode(parts[0]).decode('utf8') - task_id = parts[1].replace(b'\n', b'') - self.run_code(code, task_id) except binascii.Error: self.logger.error("Invalid base64 code string received!") + return + + task_id = parts[1].replace(b'\n', b'') + self.run_code(code, task_id) def _on_socket_disconnect(self, socket): def on_socket_disconnect_handler(): From d901ce05a6fcc06522c26454c825731fb384d7af Mon Sep 17 00:00:00 2001 From: Alexander Kozlovsky Date: Mon, 2 May 2022 09:01:36 +0200 Subject: [PATCH 15/16] Report task_id and code on crash in the executor --- src/tribler/gui/code_executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tribler/gui/code_executor.py b/src/tribler/gui/code_executor.py index 0b961641e67..7df67106b92 100644 --- a/src/tribler/gui/code_executor.py +++ b/src/tribler/gui/code_executor.py @@ -60,7 +60,7 @@ def run_code(self, code, task_id): pass if self.shell.last_traceback: - self.on_crash(self.shell.last_traceback) + self.on_crash(f'{self.shell.last_traceback}\n\ntask_id: {task_id!r}\ncode:\n{code}\n\n(end of code)') return self.logger.info("Code execution with task %s finished:", task_id) From 8f645138bc4ba62f984c1931034547606c4e4ea4 Mon Sep 17 00:00:00 2001 From: Alexander Kozlovsky Date: Mon, 2 May 2022 10:29:19 +0200 Subject: [PATCH 16/16] Delay CodeExecutor start until the Tribler core is connected to GUI --- src/tribler/gui/code_executor.py | 15 +++++++++++++-- src/tribler/gui/tribler_app.py | 2 ++ src/tribler/gui/tribler_window.py | 4 ++-- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/tribler/gui/code_executor.py b/src/tribler/gui/code_executor.py index 7df67106b92..68ecb2477f1 100644 --- a/src/tribler/gui/code_executor.py +++ b/src/tribler/gui/code_executor.py @@ -27,15 +27,26 @@ class CodeExecutor: def __init__(self, port, shell_variables=None): self.logger = logging.getLogger(self.__class__.__name__) + self.port = port self.tcp_server = QTcpServer() self.sockets = [] self.stack_trace = None - if not self.tcp_server.listen(port=port): + self.shell = Console(locals=shell_variables or {}, logger=self.logger) + self.started = False + + def on_core_connected(self, _): + self.logger.info('Core connected, starting code executor') + + if self.started: + return + + if not self.tcp_server.listen(port=self.port): self.logger.error("Unable to start code execution socket! Error: %s", self.tcp_server.errorString()) else: connect(self.tcp_server.newConnection, self._on_new_connection) - self.shell = Console(locals=shell_variables or {}, logger=self.logger) + self.started = True + self.logger.info('Code executor started') def _on_new_connection(self): self.logger.info("CodeExecutor has new connection") diff --git a/src/tribler/gui/tribler_app.py b/src/tribler/gui/tribler_app.py index 51eeddb12b8..97a5ddadf19 100644 --- a/src/tribler/gui/tribler_app.py +++ b/src/tribler/gui/tribler_app.py @@ -50,8 +50,10 @@ def parse_sys_args(self, args): variables.update(locals()) variables['window'] = self.tribler_window self.code_executor = CodeExecutor(5500, shell_variables=variables) + connect(self.tribler_window.events_manager.core_connected, self.code_executor.on_core_connected) connect(self.tribler_window.tribler_crashed, self.code_executor.on_crash) + if '--testnet' in sys.argv[1:]: os.environ['TESTNET'] = "YES" if '--trustchain-testnet' in sys.argv[1:]: diff --git a/src/tribler/gui/tribler_window.py b/src/tribler/gui/tribler_window.py index 4d8b59ebefa..943a331996b 100644 --- a/src/tribler/gui/tribler_window.py +++ b/src/tribler/gui/tribler_window.py @@ -199,8 +199,8 @@ def __init__( self.core_env = core_env error_handler = ErrorHandler(self) - events_manager = EventRequestManager(api_port, api_key, error_handler) - self.core_manager = CoreManager(self.root_state_dir, api_port, api_key, app_manager, events_manager) + self.events_manager = EventRequestManager(api_port, api_key, error_handler) + self.core_manager = CoreManager(self.root_state_dir, api_port, api_key, app_manager, self.events_manager) self.version_history = VersionHistory(self.root_state_dir) self.upgrade_manager = UpgradeManager(self.version_history) self.pending_requests = {}