Merge branch 'release/7.13' into main

.github/workflows/build_ubuntu.yml

@@ -59,6 +59,13 @@ jobs:
           fetch-depth: 0
           ref: ${{inputs.ref || github.ref}}
 
+      - name: Install Ubuntu dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get -y install debhelper devscripts
+          # for qt:
+          sudo apt-get -y install libxcb-xinerama0-dev libqt5x11extras5 libgirepository1.0-dev
+
       - name: Create python environment
         id: pyenv
         uses: ./.github/actions/pyenv
@@ -69,13 +76,6 @@ jobs:
 
       - uses: ./.github/actions/save_git_info
 
-      - name: Install Ubuntu dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get -y install debhelper devscripts
-          # for qt:
-          sudo apt-get -y install libxcb-xinerama0-dev libqt5x11extras5
-
       - name: Run build script
         timeout-minutes: 10
         env:

requirements-build.txt

@@ -1,11 +1,12 @@
 -r requirements.txt
 
-PyInstaller==5.1; sys_platform != 'darwin'
+PyInstaller==5.11.0; sys_platform != 'darwin'
 
 setuptools==65.5.1; sys_platform == 'darwin'
 text-unidecode==1.3; sys_platform == 'darwin'
 
 defusedxml==0.7.1; sys_platform == 'linux2' or sys_platform == 'linux'
 markupsafe==2.0.1; sys_platform == 'linux2' or sys_platform == 'linux'
+PyGObject==3.44.1; sys_platform == 'linux2' or sys_platform == 'linux'
 
 requests==2.31.0

requirements-test.txt

@@ -16,3 +16,6 @@ coverage==7.2.7
 looptime==0.2
 
 scipy==1.10.1
+
+# for pyqt5 test
+pytest-qt==4.2.0

src/tribler/core/components/conftest.py

@@ -1,6 +1,7 @@
 from unittest.mock import MagicMock
 
 import pytest
+from aiohttp.abc import Application
 from ipv8.keyvault.private.libnaclkey import LibNaCLSK
 from ipv8.util import succeed
 
@@ -10,6 +11,7 @@
 from tribler.core.components.libtorrent.settings import LibtorrentSettings
 from tribler.core.components.libtorrent.torrentdef import TorrentDef
 from tribler.core.components.metadata_store.db.store import MetadataStore
+from tribler.core.components.restapi.rest.rest_manager import error_middleware
 from tribler.core.config.tribler_config import TriblerConfig
 from tribler.core.tests.tools.common import TESTS_DATA_DIR
 from tribler.core.utilities.path_util import Path
@@ -130,3 +132,10 @@ async def download_manager(tmp_path_factory):
     yield download_manager
 
     await download_manager.shutdown()
+
+
+@pytest.fixture(name='web_app')
+async def web_app_fixture():
+    app = Application(middlewares=[error_middleware])
+    yield app
+    await app.shutdown()

src/tribler/core/components/knowledge/restapi/tests/test_knowledge_endpoint.py

@@ -27,11 +27,9 @@ def knowledge_endpoint(knowledge_db):
 
 
 @pytest.fixture
-def rest_api(event_loop, aiohttp_client, knowledge_endpoint):
-    app = Application()
-    app.add_subapp('/knowledge', knowledge_endpoint.app)
-    yield event_loop.run_until_complete(aiohttp_client(app))
-    app.shutdown()
+def rest_api(web_app, event_loop, aiohttp_client, knowledge_endpoint):
+    web_app.add_subapp('/knowledge', knowledge_endpoint.app)
+    yield event_loop.run_until_complete(aiohttp_client(web_app))
 
 
 def tag_to_statement(tag: str) -> Dict:

src/tribler/core/components/knowledge/rules/knowledge_rules_processor.py

@@ -60,6 +60,8 @@ def __init__(self, notifier: Notifier, db: KnowledgeDatabase, mds: MetadataStore
         self.queue_batch_size = queue_batch_size
         self.queue_max_size = queue_max_size
 
+        self._last_warning_time = 0
+
         # this queue is used to be able to process entities supplied from another thread.
         self.queue: queue.Queue[TorrentTitle] = queue.Queue(maxsize=self.queue_max_size)
 
@@ -135,8 +137,8 @@ def query(_start, _end):
         self.set_last_processed_torrent_id(end)
 
         duration = time.time() - start_time
-        self.logger.info(
-            f'[Batch] Processed: {processed} titles. Added: {added} tags. Duration: {duration:.3f} seconds.')
+        message = f'[Batch] Processed: {processed} titles. Added: {added} tags. Duration: {duration:.3f} seconds.'
+        self.logger.info(message)
 
         is_finished = end >= max_row_id
         if is_finished:
@@ -161,8 +163,8 @@ async def process_queue(self) -> int:
 
         if processed:
             duration = time.time() - start_time
-            self.logger.info(
-                f'[Queue] Processed: {processed} titles. Added: {added} tags. Duration: {duration:.3f} seconds.')
+            message = f'[Queue] Processed: {processed} titles. Added: {added} tags. Duration: {duration:.3f} seconds.'
+            self.logger.info(message)
         return processed
 
     def put_entity_to_the_queue(self, infohash: Optional[bytes] = None, title: Optional[str] = None):
@@ -174,7 +176,11 @@ def put_entity_to_the_queue(self, infohash: Optional[bytes] = None, title: Optio
         try:
             self.queue.put_nowait(TorrentTitle(infohash, title))
         except queue.Full:
-            self.logger.warning('Queue is full')
+            now = time.time()
+            time_passed = now - self._last_warning_time
+            if time_passed > 5:  # sec
+                self.logger.warning('Queue is full')
+                self._last_warning_time = now
 
     @force_switch
     async def process_torrent_title(self, infohash: Optional[bytes] = None, title: Optional[str] = None) -> int:

src/tribler/core/components/libtorrent/restapi/tests/test_downloads_endpoint.py

@@ -19,14 +19,10 @@
 
 
 @pytest.fixture
-def rest_api(event_loop, aiohttp_client, mock_dlmgr, metadata_store):  # pylint: disable=unused-argument
-
+def rest_api(web_app, event_loop, aiohttp_client, mock_dlmgr, metadata_store):
     endpoint = DownloadsEndpoint(mock_dlmgr, metadata_store=metadata_store)
-
-    app = Application(middlewares=[error_middleware])
-    app.add_subapp('/downloads', endpoint.app)
-    yield event_loop.run_until_complete(aiohttp_client(app))
-    app.shutdown()
+    web_app.add_subapp('/downloads', endpoint.app)
+    yield event_loop.run_until_complete(aiohttp_client(web_app))
 
 
 def get_hex_infohash(tdef):

src/tribler/core/components/libtorrent/restapi/tests/test_libtorrent_endpoint.py

@@ -15,11 +15,9 @@ def endpoint(mock_dlmgr, mock_lt_session):
 
 
 @pytest.fixture
-def rest_api(event_loop, aiohttp_client, endpoint):  # pylint: disable=unused-argument
-    app = Application(middlewares=[error_middleware])
-    app.add_subapp('/libtorrent', endpoint.app)
-    yield event_loop.run_until_complete(aiohttp_client(app))
-    app.shutdown()
+def rest_api(web_app, event_loop, aiohttp_client, endpoint):
+    web_app.add_subapp('/libtorrent', endpoint.app)
+    yield event_loop.run_until_complete(aiohttp_client(web_app))
 
 
 @pytest.fixture

src/tribler/core/components/libtorrent/restapi/tests/test_torrentinfo_endpoint.py

@@ -51,11 +51,9 @@ def endpoint(download_manager):
 
 
 @pytest.fixture
-def rest_api(event_loop, aiohttp_client, endpoint):  # pylint: disable=unused-argument
-    app = Application(middlewares=[error_middleware])
-    app.add_subapp('/torrentinfo', endpoint.app)
-    yield event_loop.run_until_complete(aiohttp_client(app))
-    app.shutdown()
+def rest_api(web_app, event_loop, aiohttp_client, endpoint):
+    web_app.add_subapp('/torrentinfo', endpoint.app)
+    yield event_loop.run_until_complete(aiohttp_client(web_app))
 
 
 async def test_get_torrentinfo_escaped_characters(tmp_path, rest_api):

src/tribler/core/components/metadata_store/restapi/metadata_endpoint.py

@@ -1,10 +1,11 @@
 from binascii import unhexlify
+from typing import Optional
 
 from aiohttp import ContentTypeError, web
 from aiohttp_apispec import docs
 from ipv8.REST.base_endpoint import HTTP_BAD_REQUEST, HTTP_NOT_FOUND
 from ipv8.REST.schema import schema
-from marshmallow.fields import Integer, String
+from marshmallow.fields import Boolean
 from pony.orm import db_session
 
 from tribler.core.components.metadata_store.db.orm_bindings.channel_node import LEGACY_ENTRY
@@ -50,7 +51,7 @@ class MetadataEndpoint(MetadataEndpointBase, UpdateEntryMixin):
     #          /<public_key>
     """
 
-    def __init__(self, torrent_checker: TorrentChecker, *args, **kwargs):
+    def __init__(self, torrent_checker: Optional[TorrentChecker], *args, **kwargs):
         MetadataEndpointBase.__init__(self, *args, **kwargs)
         self.torrent_checker = torrent_checker
 
@@ -195,22 +196,10 @@ async def get_channel_entries(self, request):
             200: {
                 'schema': schema(
                     HealthCheckResponse={
-                        'tracker': schema(
-                            HealthCheck={'seeders': Integer, 'leechers': Integer, 'infohash': String, 'error': String}
-                        )
+                        'checking': Boolean()
                     }
                 ),
                 'examples': [
-                    {
-                        "health": {
-                            "http://mytracker.com:80/announce": {
-                                "seeders": 43,
-                                "leechers": 20,
-                                "infohash": "97d2d8f5d37e56cfaeaae151d55f05b077074779",
-                            },
-                            "http://nonexistingtracker.com:80/announce": {"error": "timeout"},
-                        }
-                    },
                     {'checking': 1},
                 ],
             }
@@ -223,7 +212,10 @@ async def get_torrent_health(self, request):
         except ValueError as e:
             return RESTResponse({"error": f"Error processing timeout parameter: {e}"}, status=HTTP_BAD_REQUEST)
 
+        if self.torrent_checker is None:
+            return RESTResponse({'checking': False})
+
         infohash = unhexlify(request.match_info['infohash'])
         check_coro = self.torrent_checker.check_torrent_health(infohash, timeout=timeout, scrape_now=True)
         self.async_group.add_task(check_coro)
-        return RESTResponse({'checking': '1'})
+        return RESTResponse({'checking': True})

src/tribler/core/components/metadata_store/restapi/tests/test_channels_endpoint.py

@@ -36,7 +36,7 @@
 
 
 @pytest.fixture
-def rest_api(event_loop, aiohttp_client, mock_dlmgr, metadata_store, knowledge_db):  # pylint: disable=unused-argument
+def rest_api(web_app, event_loop, aiohttp_client, mock_dlmgr, metadata_store, knowledge_db):
     mock_gigachannel_manager = Mock()
     mock_gigachannel_community = Mock()
 
@@ -51,11 +51,9 @@ def return_exc(*args, **kwargs):
     collections_endpoint = ChannelsEndpoint(*ep_args, **ep_kwargs)
     channels_endpoint = ChannelsEndpoint(*ep_args, **ep_kwargs)
 
-    app = Application(middlewares=[error_middleware])
-    app.add_subapp('/channels', channels_endpoint.app)
-    app.add_subapp('/collections', collections_endpoint.app)
-    yield event_loop.run_until_complete(aiohttp_client(app))
-    app.shutdown()
+    web_app.add_subapp('/channels', channels_endpoint.app)
+    web_app.add_subapp('/collections', collections_endpoint.app)
+    yield event_loop.run_until_complete(aiohttp_client(web_app))
 
 
 async def test_get_channels(rest_api, add_fake_torrents_channels, add_subscribed_and_not_downloaded_channel, mock_dlmgr,

src/tribler/core/components/metadata_store/restapi/tests/test_metadata_endpoint.py

@@ -39,13 +39,11 @@ async def torrent_checker(mock_dlmgr, metadata_store):
 
 
 @pytest.fixture
-def rest_api(event_loop, aiohttp_client, torrent_checker, metadata_store):
+def rest_api(web_app, event_loop, aiohttp_client, torrent_checker, metadata_store):
     endpoint = MetadataEndpoint(torrent_checker, metadata_store)
 
-    app = Application(middlewares=[error_middleware])
-    app.add_subapp('/metadata', endpoint.app)
-    yield event_loop.run_until_complete(aiohttp_client(app))
-    app.shutdown()
+    web_app.add_subapp('/metadata', endpoint.app)
+    yield event_loop.run_until_complete(aiohttp_client(web_app))
 
 
 async def test_update_multiple_metadata_entries(metadata_store, add_fake_torrents_channels, rest_api):
@@ -208,7 +206,7 @@ async def test_check_torrent_health(rest_api, mock_dlmgr, udp_tracker, metadata_
     infohash = b'a' * 20
     url = f'metadata/torrents/{hexlify(infohash)}/health?timeout={TORRENT_CHECK_TIMEOUT}'
     json_response = await do_request(rest_api, url)
-    assert json_response == {'checking': '1'}
+    assert json_response == {'checking': True}
 
 
 async def test_check_torrent_query(rest_api, udp_tracker, metadata_store):

src/tribler/core/components/metadata_store/restapi/tests/test_search_endpoint.py

@@ -31,12 +31,10 @@ def needle_in_haystack_mds(metadata_store):
 
 
 @pytest.fixture
-def rest_api(event_loop, needle_in_haystack_mds, aiohttp_client, knowledge_db):
+def rest_api(web_app, event_loop, needle_in_haystack_mds, aiohttp_client, knowledge_db):
     channels_endpoint = SearchEndpoint(needle_in_haystack_mds, knowledge_db=knowledge_db)
-    app = Application()
-    app.add_subapp('/search', channels_endpoint.app)
-    yield event_loop.run_until_complete(aiohttp_client(app))
-    app.shutdown()
+    web_app.add_subapp('/search', channels_endpoint.app)
+    yield event_loop.run_until_complete(aiohttp_client(web_app))
 
 
 async def test_search_wrong_mdtype(rest_api):

src/tribler/core/components/popularity/community/popularity_community.py

@@ -62,7 +62,8 @@ def get_alive_checked_torrents(self) -> List[HealthInfo]:
             return []
 
         # Filter torrents that have seeders
-        return [health for health in self.torrent_checker.torrents_checked.values() if health.seeders > 0]
+        return [health for health in self.torrent_checker.torrents_checked.values() if
+                health.seeders > 0 and health.leechers >= 0]
 
     def gossip_random_torrents_health(self):
         """

src/tribler/core/components/restapi/rest/debug_endpoint.py

@@ -2,18 +2,19 @@
 import logging
 import sys
 from io import StringIO
+from typing import Optional
 
 import psutil
 from aiohttp import web
 from aiohttp.abc import BaseRequest
 from aiohttp_apispec import docs
 from ipv8.REST.schema import schema
-from ipv8.messaging.anonymization.community import TunnelCommunity
 from marshmallow.fields import Boolean, Float, Integer, String
 
 from tribler.core.components.reporter.exception_handler import CoreExceptionHandler
 from tribler.core.components.resource_monitor.implementation.base import ResourceMonitor
 from tribler.core.components.restapi.rest.rest_endpoint import RESTEndpoint, RESTResponse
+from tribler.core.components.tunnel.community.tunnel_community import TriblerTunnelCommunity
 from tribler.core.exceptions import TriblerCoreTestException
 from tribler.core.utilities.instrumentation import WatchDog
 from tribler.core.utilities.osutils import get_root_state_directory
@@ -46,9 +47,9 @@ class DebugEndpoint(RESTEndpoint):
     def __init__(self,
                  state_dir: Path,
                  log_dir: Path,
-                 tunnel_community: TunnelCommunity = None,
-                 resource_monitor: ResourceMonitor = None,
-                 core_exception_handler: CoreExceptionHandler = None):
+                 tunnel_community: Optional[TriblerTunnelCommunity] = None,
+                 resource_monitor: Optional[ResourceMonitor] = None,
+                 core_exception_handler: Optional[CoreExceptionHandler] = None):
         super().__init__()
         self.logger = logging.getLogger(self.__class__.__name__)
         self.state_dir = state_dir
@@ -88,10 +89,12 @@ def setup_routes(self):
         }
     )
     async def get_circuit_slots(self, request):
+        random_slots = self.tunnel_community.random_slots if self.tunnel_community else []
+        competing_slots = self.tunnel_community.competing_slots if self.tunnel_community else []
         return RESTResponse({
             "slots": {
-                "random": self.tunnel_community.random_slots,
-                "competing": self.tunnel_community.competing_slots
+                "random": random_slots,
+                "competing": competing_slots
             }
         })
 

src/tribler/core/components/restapi/rest/tests/test_create_torrent_endpoint.py

@@ -17,11 +17,9 @@ def endpoint():
 
 
 @pytest.fixture
-def rest_api(event_loop, aiohttp_client, endpoint):  # pylint: disable=unused-argument
-    app = Application(middlewares=[error_middleware])
-    app.add_subapp('/createtorrent', endpoint.app)
-    yield event_loop.run_until_complete(aiohttp_client(app))
-    app.shutdown()
+def rest_api(web_app, event_loop, aiohttp_client, endpoint):
+    web_app.add_subapp('/createtorrent', endpoint.app)
+    yield event_loop.run_until_complete(aiohttp_client(web_app))
 
 
 async def test_create_torrent(rest_api, tmp_path, endpoint):

src/tribler/core/components/restapi/rest/tests/test_debug_endpoint.py

@@ -39,13 +39,10 @@ async def core_resource_monitor(tmp_path):
 
 
 @pytest.fixture
-def rest_api(event_loop, aiohttp_client, mock_tunnel_community, endpoint):
+def rest_api(web_app, event_loop, aiohttp_client, mock_tunnel_community, endpoint):
     endpoint.tunnel_community = mock_tunnel_community
-
-    app = Application(middlewares=[error_middleware])
-    app.add_subapp('/debug', endpoint.app)
-    yield event_loop.run_until_complete(aiohttp_client(app))
-    app.shutdown()
+    web_app.add_subapp('/debug', endpoint.app)
+    yield event_loop.run_until_complete(aiohttp_client(web_app))
 
 
 async def test_get_slots(rest_api, mock_tunnel_community):