Merge pull request #7522 from drew2a/fix/rest_api_fixture

Make `rest_api` fixture async

src/tribler/core/components/conftest.py

@@ -1,7 +1,6 @@
 from unittest.mock import MagicMock
 
 import pytest
-from aiohttp.abc import Application
 from ipv8.keyvault.private.libnaclkey import LibNaCLSK
 from ipv8.util import succeed
 
@@ -11,7 +10,6 @@
 from tribler.core.components.libtorrent.settings import LibtorrentSettings
 from tribler.core.components.libtorrent.torrentdef import TorrentDef
 from tribler.core.components.metadata_store.db.store import MetadataStore
-from tribler.core.components.restapi.rest.rest_manager import error_middleware
 from tribler.core.config.tribler_config import TriblerConfig
 from tribler.core.tests.tools.common import TESTS_DATA_DIR
 from tribler.core.utilities.path_util import Path
@@ -132,10 +130,3 @@ async def download_manager(tmp_path_factory):
     yield download_manager
 
     await download_manager.shutdown()
-
-
-@pytest.fixture(name='web_app')
-async def web_app_fixture():
-    app = Application(middlewares=[error_middleware])
-    yield app
-    await app.shutdown()

src/tribler/core/components/knowledge/restapi/tests/test_knowledge_endpoint.py

@@ -27,9 +27,13 @@ def knowledge_endpoint(knowledge_db):
 
 
 @pytest.fixture
-def rest_api(web_app, event_loop, aiohttp_client, knowledge_endpoint):
-    web_app.add_subapp('/knowledge', knowledge_endpoint.app)
-    yield event_loop.run_until_complete(aiohttp_client(web_app))
+async def rest_api(aiohttp_client, knowledge_endpoint):
+    app = Application()
+    app.add_subapp('/knowledge', knowledge_endpoint.app)
+
+    yield await aiohttp_client(app)
+
+    await app.shutdown()
 
 
 def tag_to_statement(tag: str) -> Dict:

src/tribler/core/components/libtorrent/restapi/tests/test_downloads_endpoint.py

@@ -19,10 +19,15 @@
 
 
 @pytest.fixture
-def rest_api(web_app, event_loop, aiohttp_client, mock_dlmgr, metadata_store):
+async def rest_api(aiohttp_client, mock_dlmgr, metadata_store):
     endpoint = DownloadsEndpoint(mock_dlmgr, metadata_store=metadata_store)
-    web_app.add_subapp('/downloads', endpoint.app)
-    yield event_loop.run_until_complete(aiohttp_client(web_app))
+    app = Application(middlewares=[error_middleware])
+    app.add_subapp('/downloads', endpoint.app)
+
+    yield await aiohttp_client(app)
+
+    await endpoint.shutdown()
+    await app.shutdown()
 
 
 def get_hex_infohash(tdef):
@@ -144,12 +149,12 @@ def test_get_extended_status_circuits(mock_extended_status):
 
 
 @unittest.mock.patch("tribler.core.components.libtorrent.restapi.downloads_endpoint.ensure_unicode",
-       Mock(side_effect=UnicodeDecodeError("", b"", 0, 0, "")))
+                     Mock(side_effect=UnicodeDecodeError("", b"", 0, 0, "")))
 def test_safe_extended_peer_info():
     """
     Test that we return the string mapped by `chr` in the case of `UnicodeDecodeError`
     """
-    extended_peer_info = download_endpoint._safe_extended_peer_info(b"abcd") # pylint: disable=protected-access
+    extended_peer_info = download_endpoint._safe_extended_peer_info(b"abcd")  # pylint: disable=protected-access
     assert extended_peer_info == "abcd"
 
 

src/tribler/core/components/libtorrent/restapi/tests/test_libtorrent_endpoint.py

@@ -10,14 +10,15 @@
 
 
 @pytest.fixture
-def endpoint(mock_dlmgr, mock_lt_session):
-    return LibTorrentEndpoint(mock_dlmgr)
+async def rest_api(aiohttp_client, mock_dlmgr, mock_lt_session):
+    endpoint = LibTorrentEndpoint(mock_dlmgr)
+    app = Application(middlewares=[error_middleware])
+    app.add_subapp('/libtorrent', endpoint.app)
 
+    yield await aiohttp_client(app)
 
-@pytest.fixture
-def rest_api(web_app, event_loop, aiohttp_client, endpoint):
-    web_app.add_subapp('/libtorrent', endpoint.app)
-    yield event_loop.run_until_complete(aiohttp_client(web_app))
+    await endpoint.shutdown()
+    await app.shutdown()
 
 
 @pytest.fixture

src/tribler/core/components/libtorrent/restapi/tests/test_torrentinfo_endpoint.py

@@ -1,14 +1,15 @@
 import json
 import shutil
 from binascii import unhexlify
-from unittest.mock import MagicMock, patch
+from unittest.mock import AsyncMock, MagicMock, patch
 from urllib.parse import quote_plus, unquote_plus
 
 import pytest
 from aiohttp.web_app import Application
 from ipv8.util import succeed
 
 from tribler.core import notifications
+from tribler.core.components.libtorrent.download_manager.download_manager import DownloadManager
 from tribler.core.components.libtorrent.restapi.torrentinfo_endpoint import TorrentInfoEndpoint
 from tribler.core.components.libtorrent.settings import DownloadDefaultsSettings, LibtorrentSettings
 from tribler.core.components.libtorrent.torrentdef import TorrentDef
@@ -25,7 +26,6 @@
 
 # pylint: disable=redefined-outer-name
 
-
 @pytest.fixture
 def download_manager(state_dir):
     dlmgr = MagicMock()
@@ -46,14 +46,15 @@ def download_manager(state_dir):
 
 
 @pytest.fixture
-def endpoint(download_manager):
-    return TorrentInfoEndpoint(download_manager)
+async def rest_api(aiohttp_client, download_manager: DownloadManager):
+    endpoint = TorrentInfoEndpoint(download_manager)
+    app = Application(middlewares=[error_middleware])
+    app.add_subapp('/torrentinfo', endpoint.app)
 
+    yield await aiohttp_client(app)
 
-@pytest.fixture
-def rest_api(web_app, event_loop, aiohttp_client, endpoint):
-    web_app.add_subapp('/torrentinfo', endpoint.app)
-    yield event_loop.run_until_complete(aiohttp_client(web_app))
+    await endpoint.shutdown()
+    await app.shutdown()
 
 
 async def test_get_torrentinfo_escaped_characters(tmp_path, rest_api):
@@ -67,7 +68,7 @@ async def test_get_torrentinfo_escaped_characters(tmp_path, rest_api):
     assert 'metainfo' in response
 
 
-async def test_get_torrentinfo(tmp_path, rest_api, endpoint: TorrentInfoEndpoint):
+async def test_get_torrentinfo(tmp_path, rest_api, download_manager: DownloadManager):
     """
     Testing whether the API returns a correct dictionary with torrent info.
     """
@@ -113,27 +114,27 @@ async def mock_http_query(*_):
         tdef = TorrentDef.load_from_memory(torrent_data)
     metainfo_dict = tdef_to_metadata_dict(TorrentDef.load_from_memory(torrent_data))
 
-    def get_metainfo(infohash, timeout=20, hops=None, url=None):
+    async def get_metainfo(infohash, timeout=20, hops=None, url=None):  # pylint: disable=unused-argument
         if hops is not None:
             hops_list.append(hops)
         assert url
         assert url == unquote_plus(path)
-        return succeed(tdef.get_metainfo())
+        return tdef.get_metainfo()
 
-    endpoint.download_manager.get_metainfo = get_metainfo
+    download_manager.get_metainfo = get_metainfo
     verify_valid_dict(await do_request(rest_api, f'torrentinfo?uri={path}', expected_code=200))
 
     path = 'magnet:?xt=urn:ed2k:354B15E68FB8F36D7CD88FF94116CDC1'  # No infohash
     await do_request(rest_api, f'torrentinfo?uri={path}', expected_code=400)
 
     path = quote_plus(f"magnet:?xt=urn:btih:{'a' * 40}&dn=test torrent")
-    endpoint.download_manager.get_metainfo = lambda *_, **__: succeed(None)
+    download_manager.get_metainfo = lambda *_, **__: succeed(None)
     await do_request(rest_api, f'torrentinfo?uri={path}', expected_code=500)
 
     # Ensure that correct torrent metadata was sent through notifier (to MetadataStore)
-    endpoint.download_manager.notifier[notifications.torrent_metadata_added].assert_called_with(metainfo_dict)
+    download_manager.notifier[notifications.torrent_metadata_added].assert_called_with(metainfo_dict)
 
-    endpoint.download_manager.get_metainfo = get_metainfo
+    download_manager.get_metainfo = get_metainfo
     verify_valid_dict(await do_request(rest_api, f'torrentinfo?uri={path}', expected_code=200))
 
     await do_request(rest_api, f'torrentinfo?uri={path}&hops=0', expected_code=200)
@@ -144,22 +145,25 @@ def get_metainfo(infohash, timeout=20, hops=None, url=None):
     path = 'http://fdsafksdlafdslkdksdlfjs9fsafasdf7lkdzz32.n38/324.torrent'
     await do_request(rest_api, f'torrentinfo?uri={path}', expected_code=500)
 
-    mock_download = MagicMock()
+    mock_download = MagicMock(
+        stop=AsyncMock(),
+        shutdown=AsyncMock()
+    )
     path = quote_plus(f'magnet:?xt=urn:btih:{hexlify(UBUNTU_1504_INFOHASH)}&dn=test torrent')
-    endpoint.download_manager.downloads = {UBUNTU_1504_INFOHASH: mock_download}
+    download_manager.downloads = {UBUNTU_1504_INFOHASH: mock_download}
     result = await do_request(rest_api, f'torrentinfo?uri={path}', expected_code=200)
     assert result["download_exists"]
 
     # Check that we do not return "downloads_exists" if the download is metainfo only download
-    endpoint.download_manager.downloads = {UBUNTU_1504_INFOHASH: mock_download}
-    endpoint.download_manager.metainfo_requests = {UBUNTU_1504_INFOHASH: [mock_download]}
+    download_manager.downloads = {UBUNTU_1504_INFOHASH: mock_download}
+    download_manager.metainfo_requests = {UBUNTU_1504_INFOHASH: [mock_download]}
     result = await do_request(rest_api, f'torrentinfo?uri={path}', expected_code=200)
     assert not result["download_exists"]
 
     # Check that we return "downloads_exists" if there is a metainfo download for the infohash,
     # but there is also a regular download for the same infohash
-    endpoint.download_manager.downloads = {UBUNTU_1504_INFOHASH: mock_download}
-    endpoint.download_manager.metainfo_requests = {UBUNTU_1504_INFOHASH: [MagicMock()]}
+    download_manager.downloads = {UBUNTU_1504_INFOHASH: mock_download}
+    download_manager.metainfo_requests = {UBUNTU_1504_INFOHASH: [MagicMock()]}
     result = await do_request(rest_api, f'torrentinfo?uri={path}', expected_code=200)
     assert result["download_exists"]
 

src/tribler/core/components/metadata_store/restapi/tests/test_channels_endpoint.py

@@ -36,7 +36,7 @@
 
 
 @pytest.fixture
-def rest_api(web_app, event_loop, aiohttp_client, mock_dlmgr, metadata_store, knowledge_db):
+async def rest_api(aiohttp_client, mock_dlmgr, metadata_store, knowledge_db):
     mock_gigachannel_manager = Mock()
     mock_gigachannel_community = Mock()
 
@@ -51,9 +51,15 @@ def return_exc(*args, **kwargs):
     collections_endpoint = ChannelsEndpoint(*ep_args, **ep_kwargs)
     channels_endpoint = ChannelsEndpoint(*ep_args, **ep_kwargs)
 
-    web_app.add_subapp('/channels', channels_endpoint.app)
-    web_app.add_subapp('/collections', collections_endpoint.app)
-    yield event_loop.run_until_complete(aiohttp_client(web_app))
+    app = Application(middlewares=[error_middleware])
+    app.add_subapp('/channels', channels_endpoint.app)
+    app.add_subapp('/collections', collections_endpoint.app)
+
+    yield await aiohttp_client(app)
+
+    await collections_endpoint.shutdown()
+    await channels_endpoint.shutdown()
+    await app.shutdown()
 
 
 async def test_get_channels(rest_api, add_fake_torrents_channels, add_subscribed_and_not_downloaded_channel, mock_dlmgr,

src/tribler/core/components/metadata_store/restapi/tests/test_metadata_endpoint.py

@@ -39,11 +39,15 @@ async def torrent_checker(mock_dlmgr, metadata_store):
 
 
 @pytest.fixture
-def rest_api(web_app, event_loop, aiohttp_client, torrent_checker, metadata_store):
+async def rest_api(aiohttp_client, torrent_checker, metadata_store):
     endpoint = MetadataEndpoint(torrent_checker, metadata_store)
 
-    web_app.add_subapp('/metadata', endpoint.app)
-    yield event_loop.run_until_complete(aiohttp_client(web_app))
+    app = Application(middlewares=[error_middleware])
+    app.add_subapp('/metadata', endpoint.app)
+    yield await aiohttp_client(app)
+
+    await endpoint.shutdown()
+    await app.shutdown()
 
 
 async def test_update_multiple_metadata_entries(metadata_store, add_fake_torrents_channels, rest_api):

src/tribler/core/components/metadata_store/restapi/tests/test_remote_query_endpoint.py

@@ -24,19 +24,18 @@ def mock_gigachannel_community():
 
 
 @pytest.fixture
-def endpoint(mock_gigachannel_community, metadata_store):
-    return RemoteQueryEndpoint(mock_gigachannel_community, metadata_store)
-
-
-@pytest.fixture
-def rest_api(event_loop, aiohttp_client, endpoint):
+async def rest_api(aiohttp_client, metadata_store, mock_gigachannel_community):
+    endpoint = RemoteQueryEndpoint(mock_gigachannel_community, metadata_store)
     app = Application(middlewares=[error_middleware])
     app.add_subapp('/remote_query', endpoint.app)
-    yield event_loop.run_until_complete(aiohttp_client(app))
-    app.shutdown()
+
+    yield await aiohttp_client(app)
+
+    await endpoint.shutdown()
+    await app.shutdown()
 
 
-async def test_create_remote_search_request(rest_api, endpoint, mock_gigachannel_community):
+async def test_create_remote_search_request(rest_api, mock_gigachannel_community):
     """
     Test that remote search call is sent on a REST API search request
     """
@@ -69,7 +68,7 @@ def mock_send(**kwargs):
     assert hexlify(sent['channel_pk']) == channel_pk
 
 
-async def test_get_channels_peers(rest_api, endpoint, metadata_store, mock_gigachannel_community):
+async def test_get_channels_peers(rest_api, metadata_store, mock_gigachannel_community):
     """
     Test getting debug info about the state of channels to peers mapping
     """

src/tribler/core/components/metadata_store/restapi/tests/test_search_endpoint.py

@@ -31,10 +31,15 @@ def needle_in_haystack_mds(metadata_store):
 
 
 @pytest.fixture
-def rest_api(web_app, event_loop, needle_in_haystack_mds, aiohttp_client, knowledge_db):
+async def rest_api(needle_in_haystack_mds, aiohttp_client, knowledge_db):
     channels_endpoint = SearchEndpoint(needle_in_haystack_mds, knowledge_db=knowledge_db)
-    web_app.add_subapp('/search', channels_endpoint.app)
-    yield event_loop.run_until_complete(aiohttp_client(web_app))
+    app = Application()
+    app.add_subapp('/search', channels_endpoint.app)
+
+    yield await aiohttp_client(app)
+
+    await channels_endpoint.shutdown()
+    await app.shutdown()
 
 
 async def test_search_wrong_mdtype(rest_api):

src/tribler/core/components/metadata_store/tests/test_channel_download.py

@@ -20,13 +20,15 @@
 CHANNEL_METADATA_UPDATED = CHANNEL_DIR / 'channel_upd.mdblob'
 
 
+# pylint: disable=redefined-outer-name
+
 @pytest.fixture
 def channel_tdef():
     return TorrentDef.load(TESTS_DATA_DIR / 'sample_channel' / 'channel_upd.torrent')
 
 
 @pytest.fixture
-async def channel_seeder(channel_tdef, tmp_path_factory):  # pylint: disable=unused-argument, redefined-outer-name
+async def channel_seeder(channel_tdef, tmp_path_factory):  # pylint: disable=unused-argument
     config = LibtorrentSettings()
     config.dht = False
     config.upnp = False
@@ -45,20 +47,20 @@ async def channel_seeder(channel_tdef, tmp_path_factory):  # pylint: disable=unu
 
 
 @pytest.fixture
-async def gigachannel_manager(metadata_store, download_manager):
-    gigachannel_manager = GigaChannelManager(
+async def gigachannel_manager(metadata_store, download_manager: DownloadManager):
+    manager = GigaChannelManager(
         state_dir=metadata_store.channels_dir.parent,
         download_manager=download_manager,
         metadata_store=metadata_store,
         notifier=MagicMock(),
     )
-    yield gigachannel_manager
-    await gigachannel_manager.shutdown()
+    yield manager
+    await manager.shutdown()
 
 
-async def test_channel_update_and_download(
-        channel_tdef, channel_seeder, metadata_store, download_manager, gigachannel_manager
-):
+async def test_channel_update_and_download(channel_tdef, channel_seeder, metadata_store,
+                                           download_manager: DownloadManager,
+                                           gigachannel_manager: GigaChannelManager):
     """
     Test whether we can successfully update a channel and download the new version
     """

src/tribler/core/components/resource_monitor/implementation/base.py

@@ -1,6 +1,7 @@
 import logging
 import time
 from collections import deque
+from typing import Optional
 
 import psutil
 
@@ -25,6 +26,7 @@ def __init__(self, history_size=1000):
 
         self.cpu_data = deque(maxlen=history_size)
         self.memory_data = deque(maxlen=history_size)
+        self.profiler: Optional = None
 
         self.process = psutil.Process()
         self.last_error = None

src/tribler/core/components/restapi/rest/debug_endpoint.py

@@ -340,6 +340,8 @@ async def start_profiler(self, _):
         }
     )
     async def stop_profiler(self, _):
+        if not self.resource_monitor.profiler:
+            return
         file_path = self.resource_monitor.profiler.stop()
         return RESTResponse({"success": True, "profiler_file": str(file_path)})