From 5923954965df5c498a30ca678b334bc27c0c95b4 Mon Sep 17 00:00:00 2001 From: Quinten Stokkink Date: Thu, 14 Dec 2023 09:37:37 +0100 Subject: [PATCH] Moved metadata_store into database and content_discovery (#7773) --- src/tribler/core/components/conftest.py | 2 +- .../content_discovery/community/cache.py | 5 +- .../community/content_discovery_community.py | 4 +- .../content_discovery/community/settings.py | 2 +- .../tests/test_content_discovery_community.py | 8 +- .../content_discovery_component.py | 6 +- .../restapi}/__init__.py | 0 .../content_discovery/restapi/schema.py | 9 + .../restapi/search_endpoint.py | 71 ++++ .../restapi}/tests/__init__.py | 0 .../restapi/tests/test_search_endpoint.py | 64 +++ .../tests}/__init__.py | 0 .../tests/test_content_discovery_component.py | 4 +- .../category_filter/__init__.py | 0 .../category_filter/category.conf | 0 .../category_filter/category.py | 6 +- .../category_filter/family_filter.py | 2 +- .../category_filter/filter_terms.filter | 0 .../category_filter/init_category.py | 0 .../category_filter/l2_filter.py | 2 +- .../category_filter/level2.regex | 0 .../category_filter}/tests/__init__.py | 0 .../category_filter/tests/test_category.py | 4 +- .../tests/test_family_filter.py | 4 +- .../tests/test_init_category.py | 4 +- .../{metadata_store => database}/conftest.py | 0 .../components/database/database_component.py | 48 ++- .../db/orm_bindings/__init__.py | 0 .../db/orm_bindings/misc.py | 0 .../db/orm_bindings/torrent_metadata.py | 8 +- .../db/orm_bindings/torrent_state.py | 0 .../db/orm_bindings/tracker_state.py | 0 .../db/serialization.py | 0 .../{metadata_store => database}/db/store.py | 14 +- .../db/tests/test_serialization.py | 2 +- .../db/tests/test_store.py | 6 +- .../db/tests/test_torrent_metadata.py | 4 +- .../db/tests/test_tracker_state.py | 0 .../restapi/__init__.py | 0 .../restapi/database_endpoint.py} | 239 +++++++++--- .../restapi/schema.py} | 6 - .../restapi/tests/__init__.py | 0 .../restapi/tests/test_database_endpoint.py | 368 ++++++++++++++++++ .../{metadata_store => database}/settings.py | 0 .../tests/__init__.py | 0 .../tests/test_channel_metadata.py | 2 +- .../tests/test_database_component.py} | 10 +- .../tests/test_metadata.py | 0 .../tests/test_timeutils.py | 2 +- .../gui_process_watcher/__init__.py | 0 .../gui_process_watcher/tests/__init__.py | 0 .../knowledge/knowledge_component.py | 4 +- .../rules/knowledge_rules_processor.py | 4 +- .../tests/test_knowledge_rules_processor.py | 4 +- .../tests/test_knowledge_component.py | 3 +- .../tests/test_torrentinfo_endpoint.py | 2 +- .../restapi/torrentinfo_endpoint.py | 2 +- .../metadata_store_component.py | 53 --- .../restapi/metadata_endpoint.py | 139 ------- .../restapi/metadata_endpoint_base.py | 93 ----- .../metadata_store/restapi/tests/conftest.py | 84 ---- .../restapi/tests/test_metadata_endpoint.py | 86 ---- .../restapi/tests/test_search_endpoint.py | 277 ------------- .../core/components/metadata_store/utils.py | 98 ----- .../restapi/rest/statistics_endpoint.py | 2 +- .../components/restapi/restapi_component.py | 18 +- .../restapi/tests/test_restapi_component.py | 4 +- src/tribler/core/components/tests/__init__.py | 0 .../tests/test_torrent_checker_component.py | 3 +- .../torrent_checker/torrent_checker.py | 4 +- .../torrent_checker/tracker_manager.py | 2 +- .../torrent_checker_component.py | 8 +- src/tribler/core/config/tribler_config.py | 2 +- src/tribler/core/start_core.py | 3 - src/tribler/core/tests/__init__.py | 0 src/tribler/core/upgrade/db8_to_db10.py | 2 +- .../core/upgrade/tests/test_upgrader.py | 4 +- src/tribler/core/upgrade/upgrade.py | 4 +- src/tribler/gui/tests/test_gui.py | 2 +- src/tribler/gui/tribler_window.py | 2 +- .../gui/widgets/channelcontentswidget.py | 4 +- src/tribler/gui/widgets/lazytableview.py | 2 +- .../gui/widgets/searchresultswidget.py | 4 +- .../gui/widgets/tablecontentdelegate.py | 7 +- src/tribler/gui/widgets/tablecontentmodel.py | 2 +- .../gui/widgets/triblertablecontrollers.py | 3 +- 86 files changed, 834 insertions(+), 1003 deletions(-) rename src/tribler/core/components/{metadata_store => content_discovery/restapi}/__init__.py (100%) create mode 100644 src/tribler/core/components/content_discovery/restapi/schema.py create mode 100644 src/tribler/core/components/content_discovery/restapi/search_endpoint.py rename src/tribler/core/components/{metadata_store/category_filter => content_discovery/restapi}/tests/__init__.py (100%) create mode 100644 src/tribler/core/components/content_discovery/restapi/tests/test_search_endpoint.py rename src/tribler/core/components/{metadata_store/db => content_discovery/tests}/__init__.py (100%) rename src/tribler/core/components/{metadata_store => database}/category_filter/__init__.py (100%) rename src/tribler/core/components/{metadata_store => database}/category_filter/category.conf (100%) rename src/tribler/core/components/{metadata_store => database}/category_filter/category.py (94%) rename src/tribler/core/components/{metadata_store => database}/category_filter/family_filter.py (97%) rename src/tribler/core/components/{metadata_store => database}/category_filter/filter_terms.filter (100%) rename src/tribler/core/components/{metadata_store => database}/category_filter/init_category.py (100%) rename src/tribler/core/components/{metadata_store => database}/category_filter/l2_filter.py (78%) rename src/tribler/core/components/{metadata_store => database}/category_filter/level2.regex (100%) rename src/tribler/core/components/{metadata_store/db => database/category_filter}/tests/__init__.py (100%) rename src/tribler/core/components/{metadata_store => database}/category_filter/tests/test_category.py (90%) rename src/tribler/core/components/{metadata_store => database}/category_filter/tests/test_family_filter.py (84%) rename src/tribler/core/components/{metadata_store => database}/category_filter/tests/test_init_category.py (64%) rename src/tribler/core/components/{metadata_store => database}/conftest.py (100%) rename src/tribler/core/components/{metadata_store => database}/db/orm_bindings/__init__.py (100%) rename src/tribler/core/components/{metadata_store => database}/db/orm_bindings/misc.py (100%) rename src/tribler/core/components/{metadata_store => database}/db/orm_bindings/torrent_metadata.py (96%) rename src/tribler/core/components/{metadata_store => database}/db/orm_bindings/torrent_state.py (100%) rename src/tribler/core/components/{metadata_store => database}/db/orm_bindings/tracker_state.py (100%) rename src/tribler/core/components/{metadata_store => database}/db/serialization.py (100%) rename src/tribler/core/components/{metadata_store => database}/db/store.py (98%) rename src/tribler/core/components/{metadata_store => database}/db/tests/test_serialization.py (91%) rename src/tribler/core/components/{metadata_store => database}/db/tests/test_store.py (97%) rename src/tribler/core/components/{metadata_store => database}/db/tests/test_torrent_metadata.py (98%) rename src/tribler/core/components/{metadata_store => database}/db/tests/test_tracker_state.py (100%) rename src/tribler/core/components/{metadata_store => database}/restapi/__init__.py (100%) rename src/tribler/core/components/{metadata_store/restapi/search_endpoint.py => database/restapi/database_endpoint.py} (52%) rename src/tribler/core/components/{metadata_store/restapi/metadata_schema.py => database/restapi/schema.py} (87%) rename src/tribler/core/components/{metadata_store => database}/restapi/tests/__init__.py (100%) create mode 100644 src/tribler/core/components/database/restapi/tests/test_database_endpoint.py rename src/tribler/core/components/{metadata_store => database}/settings.py (100%) rename src/tribler/core/components/{metadata_store => database}/tests/__init__.py (100%) rename src/tribler/core/components/{metadata_store => database}/tests/test_channel_metadata.py (94%) rename src/tribler/core/components/{metadata_store/tests/test_metadata_store_component.py => database/tests/test_database_component.py} (64%) rename src/tribler/core/components/{metadata_store => database}/tests/test_metadata.py (100%) rename src/tribler/core/components/{metadata_store => database}/tests/test_timeutils.py (89%) create mode 100644 src/tribler/core/components/gui_process_watcher/__init__.py create mode 100644 src/tribler/core/components/gui_process_watcher/tests/__init__.py delete mode 100644 src/tribler/core/components/metadata_store/metadata_store_component.py delete mode 100644 src/tribler/core/components/metadata_store/restapi/metadata_endpoint.py delete mode 100644 src/tribler/core/components/metadata_store/restapi/metadata_endpoint_base.py delete mode 100644 src/tribler/core/components/metadata_store/restapi/tests/conftest.py delete mode 100644 src/tribler/core/components/metadata_store/restapi/tests/test_metadata_endpoint.py delete mode 100644 src/tribler/core/components/metadata_store/restapi/tests/test_search_endpoint.py delete mode 100644 src/tribler/core/components/metadata_store/utils.py create mode 100644 src/tribler/core/components/tests/__init__.py create mode 100644 src/tribler/core/tests/__init__.py diff --git a/src/tribler/core/components/conftest.py b/src/tribler/core/components/conftest.py index 1f6fadea4d2..f9fb075cc4e 100644 --- a/src/tribler/core/components/conftest.py +++ b/src/tribler/core/components/conftest.py @@ -9,7 +9,7 @@ from tribler.core.components.libtorrent.download_manager.download_manager import DownloadManager from tribler.core.components.libtorrent.settings import LibtorrentSettings from tribler.core.components.libtorrent.torrentdef import TorrentDef -from tribler.core.components.metadata_store.db.store import MetadataStore +from tribler.core.components.database.db.store import MetadataStore from tribler.core.config.tribler_config import TriblerConfig from tribler.core.tests.tools.common import TESTS_DATA_DIR from tribler.core.utilities.path_util import Path diff --git a/src/tribler/core/components/content_discovery/community/cache.py b/src/tribler/core/components/content_discovery/community/cache.py index abbaca12857..e95adfbf308 100644 --- a/src/tribler/core/components/content_discovery/community/cache.py +++ b/src/tribler/core/components/content_discovery/community/cache.py @@ -1,7 +1,6 @@ from asyncio import Future from ipv8.requestcache import RandomNumberCache -from tribler.core.components.metadata_store.utils import RequestTimeoutException class SelectRequest(RandomNumberCache): @@ -25,6 +24,10 @@ def on_timeout(self): self.timeout_callback(self) +class RequestTimeoutException(Exception): + pass + + class EvaSelectRequest(SelectRequest): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/src/tribler/core/components/content_discovery/community/content_discovery_community.py b/src/tribler/core/components/content_discovery/community/content_discovery_community.py index cfb2f0648b2..d8c37520a86 100644 --- a/src/tribler/core/components/content_discovery/community/content_discovery_community.py +++ b/src/tribler/core/components/content_discovery/community/content_discovery_community.py @@ -31,8 +31,8 @@ from tribler.core.components.ipv8.eva.protocol import EVAProtocol from tribler.core.components.ipv8.eva.result import TransferResult from tribler.core.components.knowledge.community.knowledge_validator import is_valid_resource -from tribler.core.components.metadata_store.db.orm_bindings.torrent_metadata import LZ4_EMPTY_ARCHIVE, entries_to_chunk -from tribler.core.components.metadata_store.db.store import ObjState +from tribler.core.components.database.db.orm_bindings.torrent_metadata import LZ4_EMPTY_ARCHIVE, entries_to_chunk +from tribler.core.components.database.db.store import ObjState from tribler.core.components.torrent_checker.torrent_checker.dataclasses import HealthInfo from tribler.core.upgrade.tags_to_knowledge.previous_dbs.knowledge_db import ResourceType from tribler.core.utilities.pony_utils import run_threaded diff --git a/src/tribler/core/components/content_discovery/community/settings.py b/src/tribler/core/components/content_discovery/community/settings.py index 922bdbe3f8b..994464bb31d 100644 --- a/src/tribler/core/components/content_discovery/community/settings.py +++ b/src/tribler/core/components/content_discovery/community/settings.py @@ -4,7 +4,7 @@ from ipv8.community import CommunitySettings from tribler.core.components.database.db.tribler_database import TriblerDatabase -from tribler.core.components.metadata_store.db.store import MetadataStore +from tribler.core.components.database.db.store import MetadataStore from tribler.core.components.torrent_checker.torrent_checker.torrent_checker import TorrentChecker from tribler.core.utilities.notifier import Notifier diff --git a/src/tribler/core/components/content_discovery/community/tests/test_content_discovery_community.py b/src/tribler/core/components/content_discovery/community/tests/test_content_discovery_community.py index 7fdbcda4c9d..31e45eb1b9f 100644 --- a/src/tribler/core/components/content_discovery/community/tests/test_content_discovery_community.py +++ b/src/tribler/core/components/content_discovery/community/tests/test_content_discovery_community.py @@ -8,7 +8,7 @@ from asyncio import Future from binascii import hexlify from operator import attrgetter -from random import choices, randint, random +from random import choices, randint from typing import List from unittest.mock import AsyncMock, Mock, patch @@ -30,9 +30,9 @@ from tribler.core.components.database.db.layers.tests.test_knowledge_data_access_layer_base import \ Resource, TestKnowledgeAccessLayerBase from tribler.core.components.database.db.tribler_database import TriblerDatabase -from tribler.core.components.metadata_store.db.orm_bindings.torrent_metadata import LZ4_EMPTY_ARCHIVE, NEW -from tribler.core.components.metadata_store.db.serialization import CHANNEL_THUMBNAIL, NULL_KEY, REGULAR_TORRENT -from tribler.core.components.metadata_store.db.store import MetadataStore +from tribler.core.components.database.db.orm_bindings.torrent_metadata import LZ4_EMPTY_ARCHIVE, NEW +from tribler.core.components.database.db.serialization import CHANNEL_THUMBNAIL, NULL_KEY, REGULAR_TORRENT +from tribler.core.components.database.db.store import MetadataStore from tribler.core.components.content_discovery.community.content_discovery_community import ContentDiscoveryCommunity from tribler.core.components.torrent_checker.torrent_checker.torrent_checker import TorrentChecker from tribler.core.components.torrent_checker.torrent_checker.torrentchecker_session import HealthInfo diff --git a/src/tribler/core/components/content_discovery/content_discovery_component.py b/src/tribler/core/components/content_discovery/content_discovery_component.py index 19060cec81a..9647ce785b3 100644 --- a/src/tribler/core/components/content_discovery/content_discovery_component.py +++ b/src/tribler/core/components/content_discovery/content_discovery_component.py @@ -2,8 +2,8 @@ from ipv8.peerdiscovery.network import Network from tribler.core.components.component import Component from tribler.core.components.content_discovery.community.content_discovery_community import ContentDiscoveryCommunity +from tribler.core.components.database.database_component import DatabaseComponent from tribler.core.components.ipv8.ipv8_component import INFINITE, Ipv8Component -from tribler.core.components.metadata_store.metadata_store_component import MetadataStoreComponent from tribler.core.components.reporter.reporter_component import ReporterComponent from tribler.core.components.torrent_checker.torrent_checker_component import TorrentCheckerComponent @@ -18,7 +18,7 @@ async def run(self): await self.get_component(ReporterComponent) self._ipv8_component = await self.require_component(Ipv8Component) - metadata_store_component = await self.require_component(MetadataStoreComponent) + database_component = await self.require_component(DatabaseComponent) torrent_checker_component = await self.require_component(TorrentCheckerComponent) self.community = ContentDiscoveryCommunity(ContentDiscoveryCommunity.settings_class( @@ -26,7 +26,7 @@ async def run(self): endpoint = self._ipv8_component.ipv8.endpoint, network = Network(), maximum_payload_size = self.session.config.content_discovery_community.maximum_payload_size, - metadata_store=metadata_store_component.mds, + metadata_store=database_component.mds, torrent_checker=torrent_checker_component.torrent_checker, notifier=self.session.notifier )) diff --git a/src/tribler/core/components/metadata_store/__init__.py b/src/tribler/core/components/content_discovery/restapi/__init__.py similarity index 100% rename from src/tribler/core/components/metadata_store/__init__.py rename to src/tribler/core/components/content_discovery/restapi/__init__.py diff --git a/src/tribler/core/components/content_discovery/restapi/schema.py b/src/tribler/core/components/content_discovery/restapi/schema.py new file mode 100644 index 00000000000..819e8f778da --- /dev/null +++ b/src/tribler/core/components/content_discovery/restapi/schema.py @@ -0,0 +1,9 @@ +from marshmallow.fields import Integer, String + +from tribler.core.components.database.restapi.schema import MetadataParameters + + +class RemoteQueryParameters(MetadataParameters): + uuid = String() + channel_pk = String(description='Channel to query, must also define origin_id') + origin_id = Integer(default=None, description='Peer id to query, must also define channel_pk') diff --git a/src/tribler/core/components/content_discovery/restapi/search_endpoint.py b/src/tribler/core/components/content_discovery/restapi/search_endpoint.py new file mode 100644 index 00000000000..190572a59ff --- /dev/null +++ b/src/tribler/core/components/content_discovery/restapi/search_endpoint.py @@ -0,0 +1,71 @@ +from binascii import hexlify, unhexlify + +from aiohttp import web +from aiohttp_apispec import docs, querystring_schema +from marshmallow.fields import List, String + +from ipv8.REST.schema import schema +from tribler.core.components.content_discovery.community.content_discovery_community import ContentDiscoveryCommunity +from tribler.core.components.content_discovery.restapi.schema import RemoteQueryParameters +from tribler.core.components.restapi.rest.rest_endpoint import HTTP_BAD_REQUEST, MAX_REQUEST_SIZE, RESTEndpoint, \ + RESTResponse +from tribler.core.utilities.utilities import froze_it + + +@froze_it +class SearchEndpoint(RESTEndpoint): + """ + This endpoint is responsible for searching in channels and torrents present in the local Tribler database. + """ + path = '/search' + + def __init__(self, + popularity_community: ContentDiscoveryCommunity, + middlewares=(), + client_max_size=MAX_REQUEST_SIZE): + super().__init__(middlewares, client_max_size) + self.popularity_community = popularity_community + + def setup_routes(self): + self.app.add_routes([web.put('/remote', self.remote_search)]) + + @classmethod + def sanitize_parameters(cls, parameters): + sanitized = dict(parameters) + if "max_rowid" in parameters: + sanitized["max_rowid"] = int(parameters["max_rowid"]) + if "channel_pk" in parameters: + sanitized["channel_pk"] = unhexlify(parameters["channel_pk"]) + if "origin_id" in parameters: + sanitized["origin_id"] = int(parameters["origin_id"]) + return sanitized + + + @docs( + tags=['Metadata'], + summary="Perform a search for a given query.", + responses={200: { + 'schema': schema(RemoteSearchResponse={'request_uuid': String(), 'peers': List(String())})}, + "examples": { + 'Success': { + "request_uuid": "268560c0-3f28-4e6e-9d85-d5ccb0269693", + "peers": ["50e9a2ce646c373985a8e827e328830e053025c6", "107c84e5d9636c17b46c88c3ddb54842d80081b0"] + } + } + }, + ) + @querystring_schema(RemoteQueryParameters) + async def remote_search(self, request): + self._logger.info('Create remote search request') + # Query remote results from the GigaChannel Community. + # Results are returned over the Events endpoint. + try: + sanitized = self.sanitize_parameters(request.query) + except (ValueError, KeyError) as e: + return RESTResponse({"error": f"Error processing request parameters: {e}"}, status=HTTP_BAD_REQUEST) + self._logger.info(f'Parameters: {sanitized}') + + request_uuid, peers_list = self.popularity_community.send_search_request(**sanitized) + peers_mid_list = [hexlify(p.mid).decode() for p in peers_list] + + return RESTResponse({"request_uuid": str(request_uuid), "peers": peers_mid_list}) diff --git a/src/tribler/core/components/metadata_store/category_filter/tests/__init__.py b/src/tribler/core/components/content_discovery/restapi/tests/__init__.py similarity index 100% rename from src/tribler/core/components/metadata_store/category_filter/tests/__init__.py rename to src/tribler/core/components/content_discovery/restapi/tests/__init__.py diff --git a/src/tribler/core/components/content_discovery/restapi/tests/test_search_endpoint.py b/src/tribler/core/components/content_discovery/restapi/tests/test_search_endpoint.py new file mode 100644 index 00000000000..2e9f158d830 --- /dev/null +++ b/src/tribler/core/components/content_discovery/restapi/tests/test_search_endpoint.py @@ -0,0 +1,64 @@ +import uuid +from unittest.mock import Mock + +import pytest + +from tribler.core.components.content_discovery.restapi.search_endpoint import SearchEndpoint +from tribler.core.components.restapi.rest.base_api_test import do_request +from tribler.core.utilities.unicode import hexlify + + +@pytest.fixture(name="mock_content_discovery_community") +def fixture_mock_content_discovery_community(): + return Mock() + + +@pytest.fixture(name="endpoint") +def fixture_endpoint(mock_content_discovery_community): + return SearchEndpoint(mock_content_discovery_community) + + +async def test_create_remote_search_request(rest_api, mock_content_discovery_community): + """ + Test that remote search call is sent on a REST API search request + """ + sent = {} + peers = [] + request_uuid = uuid.uuid4() + + def mock_send(**kwargs): + sent.update(kwargs) + return request_uuid, peers + + # Test querying for keywords + mock_content_discovery_community.send_search_request = mock_send + search_txt = "foo" + await do_request( + rest_api, + f'search/remote?txt_filter={search_txt}&max_rowid=1', + request_type="PUT", + expected_code=200, + expected_json={"request_uuid": str(request_uuid), "peers": peers}, + ) + assert sent['txt_filter'] == search_txt + sent.clear() + + # Test querying channel data by public key, e.g. for channel preview purposes + channel_pk = "ff" + await do_request( + rest_api, f'search/remote?channel_pk={channel_pk}&metadata_type=torrent', request_type="PUT", expected_code=200 + ) + assert hexlify(sent['channel_pk']) == channel_pk + + +async def test_create_remote_search_request_illegal(rest_api): + """ + Test that remote search call is sent on a REST API search request + """ + response = await do_request( + rest_api, + 'search/remote?origin_id=a', + request_type="PUT", + expected_code=400 + ) + assert "error" in response diff --git a/src/tribler/core/components/metadata_store/db/__init__.py b/src/tribler/core/components/content_discovery/tests/__init__.py similarity index 100% rename from src/tribler/core/components/metadata_store/db/__init__.py rename to src/tribler/core/components/content_discovery/tests/__init__.py diff --git a/src/tribler/core/components/content_discovery/tests/test_content_discovery_component.py b/src/tribler/core/components/content_discovery/tests/test_content_discovery_component.py index 5afb0b4bee8..62513c35c47 100644 --- a/src/tribler/core/components/content_discovery/tests/test_content_discovery_component.py +++ b/src/tribler/core/components/content_discovery/tests/test_content_discovery_component.py @@ -3,7 +3,6 @@ from tribler.core.components.key.key_component import KeyComponent from tribler.core.components.knowledge.knowledge_component import KnowledgeComponent from tribler.core.components.libtorrent.libtorrent_component import LibtorrentComponent -from tribler.core.components.metadata_store.metadata_store_component import MetadataStoreComponent from tribler.core.components.content_discovery.content_discovery_component import ContentDiscoveryComponent from tribler.core.components.session import Session from tribler.core.components.socks_servers.socks_servers_component import SocksServersComponent @@ -15,8 +14,7 @@ async def test_content_discovery_component(tribler_config): components = [DatabaseComponent(), SocksServersComponent(), LibtorrentComponent(), TorrentCheckerComponent(), - KnowledgeComponent(), MetadataStoreComponent(), KeyComponent(), Ipv8Component(), - ContentDiscoveryComponent()] + KnowledgeComponent(), KeyComponent(), Ipv8Component(), ContentDiscoveryComponent()] async with Session(tribler_config, components) as session: comp = session.get_instance(ContentDiscoveryComponent) assert comp.community diff --git a/src/tribler/core/components/metadata_store/category_filter/__init__.py b/src/tribler/core/components/database/category_filter/__init__.py similarity index 100% rename from src/tribler/core/components/metadata_store/category_filter/__init__.py rename to src/tribler/core/components/database/category_filter/__init__.py diff --git a/src/tribler/core/components/metadata_store/category_filter/category.conf b/src/tribler/core/components/database/category_filter/category.conf similarity index 100% rename from src/tribler/core/components/metadata_store/category_filter/category.conf rename to src/tribler/core/components/database/category_filter/category.conf diff --git a/src/tribler/core/components/metadata_store/category_filter/category.py b/src/tribler/core/components/database/category_filter/category.py similarity index 94% rename from src/tribler/core/components/metadata_store/category_filter/category.py rename to src/tribler/core/components/database/category_filter/category.py index f1109340d30..1a8976ba944 100644 --- a/src/tribler/core/components/metadata_store/category_filter/category.py +++ b/src/tribler/core/components/database/category_filter/category.py @@ -7,12 +7,12 @@ import re from functools import cmp_to_key -from tribler.core.components.metadata_store.category_filter.family_filter import default_xxx_filter -from tribler.core.components.metadata_store.category_filter.init_category import getCategoryInfo +from tribler.core.components.database.category_filter.family_filter import default_xxx_filter +from tribler.core.components.database.category_filter.init_category import getCategoryInfo from tribler.core.utilities.install_dir import get_lib_path from tribler.core.utilities.unicode import recursive_unicode -CATEGORY_CONFIG_FILE = get_lib_path() / 'components/metadata_store/category_filter/category.conf' +CATEGORY_CONFIG_FILE = get_lib_path() / 'components/database/category_filter/category.conf' def cmp_rank(a, b): diff --git a/src/tribler/core/components/metadata_store/category_filter/family_filter.py b/src/tribler/core/components/database/category_filter/family_filter.py similarity index 97% rename from src/tribler/core/components/metadata_store/category_filter/family_filter.py rename to src/tribler/core/components/database/category_filter/family_filter.py index 7c760ee0d11..9d67b137230 100644 --- a/src/tribler/core/components/metadata_store/category_filter/family_filter.py +++ b/src/tribler/core/components/database/category_filter/family_filter.py @@ -10,7 +10,7 @@ WORDS_REGEXP = re.compile('[a-zA-Z0-9]+') -termfilename = get_lib_path() / 'components' / 'metadata_store' / 'category_filter' / 'filter_terms.filter' +termfilename = get_lib_path() / 'components' / 'database' / 'category_filter' / 'filter_terms.filter' def initTerms(filename): diff --git a/src/tribler/core/components/metadata_store/category_filter/filter_terms.filter b/src/tribler/core/components/database/category_filter/filter_terms.filter similarity index 100% rename from src/tribler/core/components/metadata_store/category_filter/filter_terms.filter rename to src/tribler/core/components/database/category_filter/filter_terms.filter diff --git a/src/tribler/core/components/metadata_store/category_filter/init_category.py b/src/tribler/core/components/database/category_filter/init_category.py similarity index 100% rename from src/tribler/core/components/metadata_store/category_filter/init_category.py rename to src/tribler/core/components/database/category_filter/init_category.py diff --git a/src/tribler/core/components/metadata_store/category_filter/l2_filter.py b/src/tribler/core/components/database/category_filter/l2_filter.py similarity index 78% rename from src/tribler/core/components/metadata_store/category_filter/l2_filter.py rename to src/tribler/core/components/database/category_filter/l2_filter.py index 21257fdd789..d62ae30550f 100644 --- a/src/tribler/core/components/metadata_store/category_filter/l2_filter.py +++ b/src/tribler/core/components/database/category_filter/l2_filter.py @@ -4,7 +4,7 @@ # !ACHTUNG! We must first read the line into a file, then release the lock, and only then pass it to regex compiler. # Otherwise, there is an annoying race condition that reads in an empty file! -with open(get_lib_path() / 'components' / 'metadata_store' / 'category_filter' / 'level2.regex', encoding="utf-8") as f: +with open(get_lib_path() / 'components' / 'database' / 'category_filter' / 'level2.regex', encoding="utf-8") as f: regex = f.read().strip() stoplist_expression = re.compile(regex, re.IGNORECASE) diff --git a/src/tribler/core/components/metadata_store/category_filter/level2.regex b/src/tribler/core/components/database/category_filter/level2.regex similarity index 100% rename from src/tribler/core/components/metadata_store/category_filter/level2.regex rename to src/tribler/core/components/database/category_filter/level2.regex diff --git a/src/tribler/core/components/metadata_store/db/tests/__init__.py b/src/tribler/core/components/database/category_filter/tests/__init__.py similarity index 100% rename from src/tribler/core/components/metadata_store/db/tests/__init__.py rename to src/tribler/core/components/database/category_filter/tests/__init__.py diff --git a/src/tribler/core/components/metadata_store/category_filter/tests/test_category.py b/src/tribler/core/components/database/category_filter/tests/test_category.py similarity index 90% rename from src/tribler/core/components/metadata_store/category_filter/tests/test_category.py rename to src/tribler/core/components/database/category_filter/tests/test_category.py index 4751d049257..a84665821b5 100644 --- a/src/tribler/core/components/metadata_store/category_filter/tests/test_category.py +++ b/src/tribler/core/components/database/category_filter/tests/test_category.py @@ -1,7 +1,7 @@ import pytest -from tribler.core.components.metadata_store.category_filter.category import Category, cmp_rank -from tribler.core.components.metadata_store.category_filter.family_filter import XXXFilter +from tribler.core.components.database.category_filter.category import Category, cmp_rank +from tribler.core.components.database.category_filter.family_filter import XXXFilter @pytest.fixture(name="xxx_filter") diff --git a/src/tribler/core/components/metadata_store/category_filter/tests/test_family_filter.py b/src/tribler/core/components/database/category_filter/tests/test_family_filter.py similarity index 84% rename from src/tribler/core/components/metadata_store/category_filter/tests/test_family_filter.py rename to src/tribler/core/components/database/category_filter/tests/test_family_filter.py index 1125ad96d1c..763199da822 100644 --- a/src/tribler/core/components/metadata_store/category_filter/tests/test_family_filter.py +++ b/src/tribler/core/components/database/category_filter/tests/test_family_filter.py @@ -1,7 +1,7 @@ import pytest -from tribler.core.components.metadata_store.category_filter.family_filter import XXXFilter -from tribler.core.components.metadata_store.category_filter.l2_filter import is_forbidden +from tribler.core.components.database.category_filter.family_filter import XXXFilter +from tribler.core.components.database.category_filter.l2_filter import is_forbidden @pytest.fixture diff --git a/src/tribler/core/components/metadata_store/category_filter/tests/test_init_category.py b/src/tribler/core/components/database/category_filter/tests/test_init_category.py similarity index 64% rename from src/tribler/core/components/metadata_store/category_filter/tests/test_init_category.py rename to src/tribler/core/components/database/category_filter/tests/test_init_category.py index 20da1bf0c4f..53fb907810c 100644 --- a/src/tribler/core/components/metadata_store/category_filter/tests/test_init_category.py +++ b/src/tribler/core/components/database/category_filter/tests/test_init_category.py @@ -1,5 +1,5 @@ -from tribler.core.components.metadata_store.category_filter.category import CATEGORY_CONFIG_FILE -from tribler.core.components.metadata_store.category_filter.init_category import INIT_FUNC_DICT, getCategoryInfo +from tribler.core.components.database.category_filter.category import CATEGORY_CONFIG_FILE +from tribler.core.components.database.category_filter.init_category import INIT_FUNC_DICT, getCategoryInfo def test_split_list(): diff --git a/src/tribler/core/components/metadata_store/conftest.py b/src/tribler/core/components/database/conftest.py similarity index 100% rename from src/tribler/core/components/metadata_store/conftest.py rename to src/tribler/core/components/database/conftest.py diff --git a/src/tribler/core/components/database/database_component.py b/src/tribler/core/components/database/database_component.py index 92c300adcb8..a9aed1c286d 100644 --- a/src/tribler/core/components/database/database_component.py +++ b/src/tribler/core/components/database/database_component.py @@ -1,5 +1,9 @@ +from tribler.core import notifications from tribler.core.components.component import Component +from tribler.core.components.database.db.store import MetadataStore from tribler.core.components.database.db.tribler_database import TriblerDatabase +from tribler.core.components.key.key_component import KeyComponent +from tribler.core.components.knowledge.rules.knowledge_rules_processor import KnowledgeRulesProcessor from tribler.core.utilities.simpledefs import STATEDIR_DB_DIR @@ -7,17 +11,57 @@ class DatabaseComponent(Component): tribler_should_stop_on_component_error = True db: TriblerDatabase = None + mds: MetadataStore = None # TODO: legacy, should be merged into ``db`` async def run(self): await super().run() - db_path = self.session.config.state_dir / STATEDIR_DB_DIR / "tribler.db" - if self.session.config.gui_test_mode: + config = self.session.config + db_path = config.state_dir / STATEDIR_DB_DIR / "tribler.db" + if config.gui_test_mode: db_path = ":memory:" self.db = TriblerDatabase(str(db_path)) + # TODO: merge the code below into the TriblerDatabase + + channels_dir = config.chant.get_path_as_absolute('channels_dir', config.state_dir) + chant_testnet = config.general.testnet or config.chant.testnet + + metadata_db_name = 'metadata.db' + if chant_testnet: + metadata_db_name = 'metadata_testnet.db' + elif config.gui_test_mode: # Avoid interfering with the main database in test mode + # Note we don't use in-memory database in core test mode, because MDS uses threads, + # and SQLite creates a different in-memory DB for each connection by default. + # To change this behaviour, we have to use url-based SQLite initialization syntax, + # which is not supported by PonyORM yet. + metadata_db_name = 'metadata_gui_test.db' + + database_path = config.state_dir / STATEDIR_DB_DIR / metadata_db_name + + # Make sure that we start with a clean metadata database when in GUI mode every time. + if config.gui_test_mode and database_path.exists(): + self.logger.info("Wiping metadata database in GUI test mode") + database_path.unlink(missing_ok=True) + + key_component = await self.require_component(KeyComponent) + + metadata_store = MetadataStore( + database_path, + channels_dir, + key_component.primary_key, + notifier=self.session.notifier, + disable_sync=config.gui_test_mode, + tag_processor_version=KnowledgeRulesProcessor.version + ) + self.mds = metadata_store + self.session.notifier.add_observer(notifications.torrent_metadata_added, + metadata_store.TorrentMetadata.add_ffa_from_dict) + async def shutdown(self): await super().shutdown() if self.db: self.db.shutdown() + if self.mds: + self.mds.shutdown() diff --git a/src/tribler/core/components/metadata_store/db/orm_bindings/__init__.py b/src/tribler/core/components/database/db/orm_bindings/__init__.py similarity index 100% rename from src/tribler/core/components/metadata_store/db/orm_bindings/__init__.py rename to src/tribler/core/components/database/db/orm_bindings/__init__.py diff --git a/src/tribler/core/components/metadata_store/db/orm_bindings/misc.py b/src/tribler/core/components/database/db/orm_bindings/misc.py similarity index 100% rename from src/tribler/core/components/metadata_store/db/orm_bindings/misc.py rename to src/tribler/core/components/database/db/orm_bindings/misc.py diff --git a/src/tribler/core/components/metadata_store/db/orm_bindings/torrent_metadata.py b/src/tribler/core/components/database/db/orm_bindings/torrent_metadata.py similarity index 96% rename from src/tribler/core/components/metadata_store/db/orm_bindings/torrent_metadata.py rename to src/tribler/core/components/database/db/orm_bindings/torrent_metadata.py index a65a0153193..2d25687665d 100644 --- a/src/tribler/core/components/metadata_store/db/orm_bindings/torrent_metadata.py +++ b/src/tribler/core/components/database/db/orm_bindings/torrent_metadata.py @@ -7,14 +7,12 @@ from lz4.frame import LZ4FrameCompressor from pony import orm from pony.orm import db_session -from pony.orm.core import DEFAULT from tribler.core import notifications -from tribler.core.components.metadata_store.category_filter.category import Category, default_category_filter -from tribler.core.components.metadata_store.category_filter.family_filter import default_xxx_filter -from tribler.core.components.metadata_store.db.serialization import EPOCH, REGULAR_TORRENT, TorrentMetadataPayload, \ +from tribler.core.components.database.category_filter.category import Category, default_category_filter +from tribler.core.components.database.category_filter.family_filter import default_xxx_filter +from tribler.core.components.database.db.serialization import EPOCH, REGULAR_TORRENT, TorrentMetadataPayload, \ HealthItemsPayload, time2int -from tribler.core.exceptions import InvalidChannelNodeException from tribler.core.utilities.notifier import Notifier from tribler.core.utilities.tracker_utils import get_uniformed_tracker_url from tribler.core.utilities.unicode import ensure_unicode, hexlify diff --git a/src/tribler/core/components/metadata_store/db/orm_bindings/torrent_state.py b/src/tribler/core/components/database/db/orm_bindings/torrent_state.py similarity index 100% rename from src/tribler/core/components/metadata_store/db/orm_bindings/torrent_state.py rename to src/tribler/core/components/database/db/orm_bindings/torrent_state.py diff --git a/src/tribler/core/components/metadata_store/db/orm_bindings/tracker_state.py b/src/tribler/core/components/database/db/orm_bindings/tracker_state.py similarity index 100% rename from src/tribler/core/components/metadata_store/db/orm_bindings/tracker_state.py rename to src/tribler/core/components/database/db/orm_bindings/tracker_state.py diff --git a/src/tribler/core/components/metadata_store/db/serialization.py b/src/tribler/core/components/database/db/serialization.py similarity index 100% rename from src/tribler/core/components/metadata_store/db/serialization.py rename to src/tribler/core/components/database/db/serialization.py diff --git a/src/tribler/core/components/metadata_store/db/store.py b/src/tribler/core/components/database/db/store.py similarity index 98% rename from src/tribler/core/components/metadata_store/db/store.py rename to src/tribler/core/components/database/db/store.py index 4d4663a4bc9..36e4d18fbbf 100644 --- a/src/tribler/core/components/metadata_store/db/store.py +++ b/src/tribler/core/components/database/db/store.py @@ -11,15 +11,11 @@ from pony.orm import db_session, desc, left_join, raw_sql, select from pony.orm.dbproviders.sqlite import keep_exception -from tribler.core.components.metadata_store.category_filter.l2_filter import is_forbidden -from tribler.core.components.metadata_store.db.orm_bindings import ( - misc, - torrent_metadata, - torrent_state as torrent_state_, - tracker_state, -) -from tribler.core.components.metadata_store.db.orm_bindings.torrent_metadata import NULL_KEY_SUBST -from tribler.core.components.metadata_store.db.serialization import ( +from tribler.core.components.database.category_filter.l2_filter import is_forbidden +from tribler.core.components.database.db.orm_bindings import misc, torrent_metadata, torrent_state as torrent_state_, \ + tracker_state +from tribler.core.components.database.db.orm_bindings.torrent_metadata import NULL_KEY_SUBST +from tribler.core.components.database.db.serialization import ( CHANNEL_TORRENT, COLLECTION_NODE, HealthItemsPayload, diff --git a/src/tribler/core/components/metadata_store/db/tests/test_serialization.py b/src/tribler/core/components/database/db/tests/test_serialization.py similarity index 91% rename from src/tribler/core/components/metadata_store/db/tests/test_serialization.py rename to src/tribler/core/components/database/db/tests/test_serialization.py index 9f15b40cc74..68aa2c510b5 100644 --- a/src/tribler/core/components/metadata_store/db/tests/test_serialization.py +++ b/src/tribler/core/components/database/db/tests/test_serialization.py @@ -1,6 +1,6 @@ from datetime import datetime -from tribler.core.components.metadata_store.db.serialization import TorrentMetadataPayload, int2time +from tribler.core.components.database.db.serialization import TorrentMetadataPayload def test_fix_torrent_metadata_payload(): diff --git a/src/tribler/core/components/metadata_store/db/tests/test_store.py b/src/tribler/core/components/database/db/tests/test_store.py similarity index 97% rename from src/tribler/core/components/metadata_store/db/tests/test_store.py rename to src/tribler/core/components/database/db/tests/test_store.py index f07b1cfb0a2..7e9a1cc21f0 100644 --- a/src/tribler/core/components/metadata_store/db/tests/test_store.py +++ b/src/tribler/core/components/database/db/tests/test_store.py @@ -7,12 +7,12 @@ from ipv8.keyvault.crypto import default_eccrypto from pony.orm import db_session -from tribler.core.components.metadata_store.db.orm_bindings.torrent_metadata import entries_to_chunk -from tribler.core.components.metadata_store.db.serialization import ( +from tribler.core.components.database.db.orm_bindings.torrent_metadata import entries_to_chunk +from tribler.core.components.database.db.serialization import ( SignedPayload, int2time, ) -from tribler.core.components.metadata_store.db.store import ObjState +from tribler.core.components.database.db.store import ObjState from tribler.core.utilities.pony_utils import run_threaded from tribler.core.utilities.utilities import random_infohash diff --git a/src/tribler/core/components/metadata_store/db/tests/test_torrent_metadata.py b/src/tribler/core/components/database/db/tests/test_torrent_metadata.py similarity index 98% rename from src/tribler/core/components/metadata_store/db/tests/test_torrent_metadata.py rename to src/tribler/core/components/database/db/tests/test_torrent_metadata.py index 5878823157c..a94834cb41b 100644 --- a/src/tribler/core/components/metadata_store/db/tests/test_torrent_metadata.py +++ b/src/tribler/core/components/database/db/tests/test_torrent_metadata.py @@ -8,9 +8,9 @@ from pony.orm import db_session from tribler.core.components.libtorrent.torrentdef import TorrentDef -from tribler.core.components.metadata_store.db.orm_bindings.torrent_metadata import tdef_to_metadata_dict, TODELETE, \ +from tribler.core.components.database.db.orm_bindings.torrent_metadata import tdef_to_metadata_dict, TODELETE, \ entries_to_chunk -from tribler.core.components.metadata_store.db.serialization import CHANNEL_TORRENT, REGULAR_TORRENT +from tribler.core.components.database.db.serialization import CHANNEL_TORRENT, REGULAR_TORRENT from tribler.core.tests.tools.common import TORRENT_UBUNTU_FILE from tribler.core.utilities.utilities import random_infohash diff --git a/src/tribler/core/components/metadata_store/db/tests/test_tracker_state.py b/src/tribler/core/components/database/db/tests/test_tracker_state.py similarity index 100% rename from src/tribler/core/components/metadata_store/db/tests/test_tracker_state.py rename to src/tribler/core/components/database/db/tests/test_tracker_state.py diff --git a/src/tribler/core/components/metadata_store/restapi/__init__.py b/src/tribler/core/components/database/restapi/__init__.py similarity index 100% rename from src/tribler/core/components/metadata_store/restapi/__init__.py rename to src/tribler/core/components/database/restapi/__init__.py diff --git a/src/tribler/core/components/metadata_store/restapi/search_endpoint.py b/src/tribler/core/components/database/restapi/database_endpoint.py similarity index 52% rename from src/tribler/core/components/metadata_store/restapi/search_endpoint.py rename to src/tribler/core/components/database/restapi/database_endpoint.py index a1570aefef9..c30ad7c29e0 100644 --- a/src/tribler/core/components/metadata_store/restapi/search_endpoint.py +++ b/src/tribler/core/components/database/restapi/database_endpoint.py @@ -1,48 +1,102 @@ import time import typing -from binascii import unhexlify, hexlify +from binascii import unhexlify from collections import defaultdict +from dataclasses import asdict from aiohttp import web from aiohttp_apispec import docs, querystring_schema -from ipv8.REST.schema import schema -from marshmallow.fields import Integer, String, List +from marshmallow.fields import Boolean, Integer, String from pony.orm import db_session +from ipv8.REST.base_endpoint import HTTP_BAD_REQUEST +from ipv8.REST.schema import schema +from tribler.core.components.database.category_filter.family_filter import default_xxx_filter from tribler.core.components.database.db.layers.knowledge_data_access_layer import ResourceType -from tribler.core.components.metadata_store.db.serialization import SNIPPET -from tribler.core.components.metadata_store.db.store import MetadataStore -from tribler.core.components.metadata_store.restapi.metadata_endpoint import MetadataEndpointBase -from tribler.core.components.metadata_store.restapi.metadata_schema import SearchMetadataParameters, MetadataSchema, \ - RemoteQueryParameters -from tribler.core.components.content_discovery.community.content_discovery_community import ContentDiscoveryCommunity -from tribler.core.components.restapi.rest.rest_endpoint import HTTP_BAD_REQUEST, RESTResponse +from tribler.core.components.database.db.serialization import REGULAR_TORRENT, SNIPPET +from tribler.core.components.database.db.store import MetadataStore +from tribler.core.components.database.db.tribler_database import TriblerDatabase +from tribler.core.components.database.restapi.schema import MetadataSchema, SearchMetadataParameters, TorrentSchema +from tribler.core.components.knowledge.rules.knowledge_rules_processor import KnowledgeRulesProcessor +from tribler.core.components.libtorrent.download_manager.download_manager import DownloadManager +from tribler.core.components.restapi.rest.rest_endpoint import MAX_REQUEST_SIZE, RESTEndpoint, RESTResponse +from tribler.core.components.torrent_checker.torrent_checker.torrent_checker import TorrentChecker from tribler.core.utilities.pony_utils import run_threaded -from tribler.core.utilities.utilities import froze_it +from tribler.core.utilities.utilities import froze_it, parse_bool +TORRENT_CHECK_TIMEOUT = 20 SNIPPETS_TO_SHOW = 3 # The number of snippets we return from the search results MAX_TORRENTS_IN_SNIPPETS = 4 # The maximum number of torrents in each snippet +# This dict is used to translate JSON fields into the columns used in Pony for _sorting_. +# id_ is not in the list because there is not index on it, so we never really want to sort on it. +json2pony_columns = { + 'category': "tags", + 'name': "title", + 'size': "size", + 'infohash': "infohash", + 'date': "torrent_date", + 'created': "torrent_date", + 'status': 'status', + 'torrents': 'num_entries', + 'votes': 'votes', + 'subscribed': 'subscribed', + 'health': 'HEALTH', +} + @froze_it -class SearchEndpoint(MetadataEndpointBase): - """ - This endpoint is responsible for searching in channels and torrents present in the local Tribler database. +class DatabaseEndpoint(RESTEndpoint): """ - path = '/search' + This is the top-level endpoint class that serves other endpoints. - def __init__(self, popularity_community: ContentDiscoveryCommunity, *args, **kwargs): - MetadataEndpointBase.__init__(self, *args, **kwargs) - self.popularity_community = popularity_community + # /metadata + # /torrents + # / + """ + path = '/metadata' + + def __init__(self, + download_manager: DownloadManager, + torrent_checker: typing.Optional[TorrentChecker], + metadata_store: MetadataStore, + tribler_db: TriblerDatabase = None, + tag_rules_processor: KnowledgeRulesProcessor = None, + middlewares=(), + client_max_size=MAX_REQUEST_SIZE): + super().__init__(middlewares, client_max_size) + self.download_manager = download_manager + self.torrent_checker = torrent_checker + self.mds = metadata_store + self.tribler_db: typing.Optional[TriblerDatabase] = tribler_db + self.tag_rules_processor: typing.Optional[KnowledgeRulesProcessor] = tag_rules_processor def setup_routes(self): - self.app.add_routes([web.get('/local', self.local_search), - web.put('/remote', self.remote_search), - web.get('/completions', self.completions)]) + self.app.add_routes( + [ + web.get('/torrents/{infohash}/health', self.get_torrent_health), + web.get('/torrents/popular', self.get_popular_torrents), + web.get('/search/local', self.local_search), + web.get('/search/completions', self.completions) + ] + ) @classmethod def sanitize_parameters(cls, parameters): - sanitized = super().sanitize_parameters(parameters) + """ + Sanitize the parameters for a request that fetches channels. + """ + sanitized = { + "first": int(parameters.get('first', 1)), + "last": int(parameters.get('last', 50)), + "sort_by": json2pony_columns.get(parameters.get('sort_by')), + "sort_desc": parse_bool(parameters.get('sort_desc', True)), + "txt_filter": parameters.get('txt_filter'), + "hide_xxx": parse_bool(parameters.get('hide_xxx', False)), + "category": parameters.get('category'), + } + if 'tags' in parameters: + sanitized['tags'] = parameters.getall('tags') if "max_rowid" in parameters: sanitized["max_rowid"] = int(parameters["max_rowid"]) if "channel_pk" in parameters: @@ -51,6 +105,118 @@ def sanitize_parameters(cls, parameters): sanitized["origin_id"] = int(parameters["origin_id"]) return sanitized + @db_session + def add_statements_to_metadata_list(self, contents_list, hide_xxx=False): + if self.tribler_db is None: + self._logger.error(f'Cannot add statements to metadata list: ' + f'tribler_db is not set in {self.__class__.__name__}') + return + for torrent in contents_list: + if torrent['type'] == REGULAR_TORRENT: + raw_statements = self.tribler_db.knowledge.get_statements( + subject_type=ResourceType.TORRENT, + subject=torrent["infohash"] + ) + statements = [asdict(stmt) for stmt in raw_statements] + if hide_xxx: + statements = [stmt for stmt in statements if not default_xxx_filter.isXXX(stmt["object"], + isFilename=False)] + torrent["statements"] = statements + + @docs( + tags=["Metadata"], + summary="Fetch the swarm health of a specific torrent.", + parameters=[ + { + 'in': 'path', + 'name': 'infohash', + 'description': 'Infohash of the download to remove', + 'type': 'string', + 'required': True, + }, + { + 'in': 'query', + 'name': 'timeout', + 'description': 'Timeout to be used in the connections to the trackers', + 'type': 'integer', + 'default': 20, + 'required': False, + }, + ], + responses={ + 200: { + 'schema': schema( + HealthCheckResponse={ + 'checking': Boolean() + } + ), + 'examples': [ + {'checking': 1}, + ], + } + }, + ) + async def get_torrent_health(self, request): + self._logger.info(f'Get torrent health request: {request}') + try: + timeout = int(request.query.get('timeout', TORRENT_CHECK_TIMEOUT)) + except ValueError as e: + return RESTResponse({"error": f"Error processing timeout parameter: {e}"}, status=HTTP_BAD_REQUEST) + + if self.torrent_checker is None: + return RESTResponse({'checking': False}) + + infohash = unhexlify(request.match_info['infohash']) + check_coro = self.torrent_checker.check_torrent_health(infohash, timeout=timeout, scrape_now=True) + self.async_group.add_task(check_coro) + return RESTResponse({'checking': True}) + + def add_download_progress_to_metadata_list(self, contents_list): + for torrent in contents_list: + if torrent['type'] == REGULAR_TORRENT: + dl = self.download_manager.get_download(unhexlify(torrent['infohash'])) + if dl is not None and dl.tdef.infohash not in self.download_manager.metainfo_requests: + torrent['progress'] = dl.get_state().get_progress() + + @docs( + tags=['Metadata'], + summary='Get the list of most popular torrents.', + responses={ + 200: { + 'schema': schema( + GetPopularTorrentsResponse={ + 'results': [TorrentSchema], + 'first': Integer(), + 'last': Integer(), + } + ) + } + }, + ) + async def get_popular_torrents(self, request): + sanitized = self.sanitize_parameters(request.query) + sanitized["metadata_type"] = REGULAR_TORRENT + sanitized["popular"] = True + + with db_session: + contents = self.mds.get_entries(**sanitized) + contents_list = [] + for entry in contents: + contents_list.append(entry.to_simple_dict()) + + if self.tag_rules_processor: + await self.tag_rules_processor.process_queue() + + self.add_download_progress_to_metadata_list(contents_list) + self.add_statements_to_metadata_list(contents_list, hide_xxx=sanitized["hide_xxx"]) + response_dict = { + "results": contents_list, + "first": sanitized['first'], + "last": sanitized['last'], + } + + return RESTResponse(response_dict) + def build_snippets(self, search_results: typing.List[typing.Dict]) -> typing.List[typing.Dict]: """ Build a list of snippets that bundle torrents describing the same content item. @@ -224,32 +390,3 @@ async def completions(self, request): # TODO: add XXX filtering for completion terms results = self.mds.get_auto_complete_terms(keywords, max_terms=5) return RESTResponse({"completions": results}) - - @docs( - tags=['Metadata'], - summary="Perform a search for a given query.", - responses={200: { - 'schema': schema(RemoteSearchResponse={'request_uuid': String(), 'peers': List(String())})}, - "examples": { - 'Success': { - "request_uuid": "268560c0-3f28-4e6e-9d85-d5ccb0269693", - "peers": ["50e9a2ce646c373985a8e827e328830e053025c6", "107c84e5d9636c17b46c88c3ddb54842d80081b0"] - } - } - }, - ) - @querystring_schema(RemoteQueryParameters) - async def remote_search(self, request): - self._logger.info('Create remote search request') - # Query remote results from the GigaChannel Community. - # Results are returned over the Events endpoint. - try: - sanitized = self.sanitize_parameters(request.query) - except (ValueError, KeyError) as e: - return RESTResponse({"error": f"Error processing request parameters: {e}"}, status=HTTP_BAD_REQUEST) - self._logger.info(f'Parameters: {sanitized}') - - request_uuid, peers_list = self.popularity_community.send_search_request(**sanitized) - peers_mid_list = [hexlify(p.mid).decode() for p in peers_list] - - return RESTResponse({"request_uuid": str(request_uuid), "peers": peers_mid_list}) diff --git a/src/tribler/core/components/metadata_store/restapi/metadata_schema.py b/src/tribler/core/components/database/restapi/schema.py similarity index 87% rename from src/tribler/core/components/metadata_store/restapi/metadata_schema.py rename to src/tribler/core/components/database/restapi/schema.py index 48cdcdd0c0d..b5b8c61c118 100644 --- a/src/tribler/core/components/metadata_store/restapi/metadata_schema.py +++ b/src/tribler/core/components/database/restapi/schema.py @@ -20,12 +20,6 @@ class SearchMetadataParameters(MetadataParameters): max_rowid = Integer(default=None, description='Only return results with rowid lesser than max_rowid') -class RemoteQueryParameters(MetadataParameters): - uuid = String() - channel_pk = String(description='Channel to query, must also define origin_id') - origin_id = Integer(default=None, description='Peer id to query, must also define channel_pk') - - class MetadataSchema(Schema): type = Integer() id = Integer() diff --git a/src/tribler/core/components/metadata_store/restapi/tests/__init__.py b/src/tribler/core/components/database/restapi/tests/__init__.py similarity index 100% rename from src/tribler/core/components/metadata_store/restapi/tests/__init__.py rename to src/tribler/core/components/database/restapi/tests/__init__.py diff --git a/src/tribler/core/components/database/restapi/tests/test_database_endpoint.py b/src/tribler/core/components/database/restapi/tests/test_database_endpoint.py new file mode 100644 index 00000000000..d38d620a37c --- /dev/null +++ b/src/tribler/core/components/database/restapi/tests/test_database_endpoint.py @@ -0,0 +1,368 @@ +import os +from binascii import unhexlify +from typing import List, Set +from unittest.mock import MagicMock, Mock, AsyncMock, patch + +import pytest +from pony.orm import db_session + +from tribler.core.components.database.category_filter.family_filter import default_xxx_filter +from tribler.core.components.database.db.layers.knowledge_data_access_layer import KnowledgeDataAccessLayer +from tribler.core.components.database.db.serialization import REGULAR_TORRENT, SNIPPET +from tribler.core.components.database.restapi.database_endpoint import DatabaseEndpoint, TORRENT_CHECK_TIMEOUT +from tribler.core.components.restapi.rest.base_api_test import do_request +from tribler.core.components.torrent_checker.torrent_checker.torrent_checker import TorrentChecker +from tribler.core.config.tribler_config import TriblerConfig +from tribler.core.utilities.unicode import hexlify +from tribler.core.utilities.utilities import random_infohash, to_fts_query + + +@pytest.fixture(name="needle_in_haystack_mds") +def fixture_needle_in_haystack_mds(metadata_store): + num_hay = 100 + with db_session: + for x in range(0, num_hay): + metadata_store.TorrentMetadata(title='hay ' + str(x), infohash=random_infohash(), public_key=b'') + metadata_store.TorrentMetadata(title='needle', infohash=random_infohash(), public_key=b'') + metadata_store.TorrentMetadata(title='needle2', infohash=random_infohash(), public_key=b'') + return metadata_store + + +@pytest.fixture(name="torrent_checker") +async def fixture_torrent_checker(mock_dlmgr, metadata_store): + # Initialize the torrent checker + config = TriblerConfig() + config.download_defaults.number_hops = 0 + tracker_manager = MagicMock() + tracker_manager.blacklist = [] + notifier = MagicMock() + torrent_checker = TorrentChecker( + config=config, + download_manager=mock_dlmgr, + tracker_manager=tracker_manager, + metadata_store=metadata_store, + notifier=notifier, + socks_listen_ports=[2000, 3000], + ) + await torrent_checker.initialize() + yield torrent_checker + await torrent_checker.shutdown() + + +@pytest.fixture(name="endpoint") +def fixture_endpoint(torrent_checker, needle_in_haystack_mds, tribler_db) -> DatabaseEndpoint: + return DatabaseEndpoint(torrent_checker.download_manager, torrent_checker, needle_in_haystack_mds, + tribler_db=tribler_db) + + +async def test_check_torrent_health(rest_api): + """ + Test the endpoint to fetch the health of a chant-managed, infohash-only torrent + """ + infohash = b'a' * 20 + url = f'metadata/torrents/{hexlify(infohash)}/health?timeout={TORRENT_CHECK_TIMEOUT}' + json_response = await do_request(rest_api, url) + assert json_response == {'checking': True} + + +async def test_check_torrent_health_no_checker(rest_api, endpoint): + """ + Test checking health without a torrent checker. + """ + endpoint.torrent_checker = None + infohash = b'a' * 20 + url = f'metadata/torrents/{hexlify(infohash)}/health?timeout={TORRENT_CHECK_TIMEOUT}' + json_response = await do_request(rest_api, url) + assert json_response == {'checking': False} + + +async def test_check_torrent_query(rest_api): + """ + Test that the endpoint responds with an error message if the timeout parameter has a wrong value + """ + infohash = b'a' * 20 + await do_request(rest_api, f"metadata/torrents/{infohash}/health?timeout=wrong_value&refresh=1", expected_code=400) + + +async def test_get_popular_torrents(rest_api, endpoint, metadata_store): + """ + Test that the endpoint responds with its known entries. + """ + fake_entry = { + "name": "Torrent Name", + "category": "", + "infohash": "ab" * 20, + "size": 1, + "num_seeders": 1234, + "num_leechers": 123, + "last_tracker_check": 17000000, + "created": 15000000, + "tag_processor_version": 1, + "type": REGULAR_TORRENT, + "id": 0, + "origin_id": 0, + "public_key": "ab" * 64, + "status": 2, + "statements": [] + } + fake_state = Mock(return_value=Mock(get_progress=Mock(return_value=0.5))) + metadata_store.get_entries = Mock(return_value=[Mock(to_simple_dict=Mock(return_value=fake_entry.copy()))]) + endpoint.tag_rules_processor = Mock(process_queue=AsyncMock()) + endpoint.download_manager.get_download = Mock(return_value=Mock(get_state=fake_state)) + response = await do_request(rest_api, "metadata/torrents/popular") + + endpoint.tag_rules_processor.process_queue.assert_called_once() + assert response == {'results': [{**fake_entry, **{"progress": 0.5}}], 'first': 1, 'last': 50} + + +async def test_get_popular_torrents_filter_xxx(rest_api, endpoint, metadata_store): + """ + Test that the endpoint responds with its known entries with xxx statements stripped, if requested. + """ + fake_entry = { + "name": next(iter(default_xxx_filter.xxx_terms)), + "category": "", + "infohash": "ab" * 20, + "size": 1, + "num_seeders": 1234, + "num_leechers": 123, + "last_tracker_check": 17000000, + "created": 15000000, + "tag_processor_version": 1, + "type": REGULAR_TORRENT, + "id": 0, + "origin_id": 0, + "public_key": "ab" * 64, + "status": 2, + "statements": [next(iter(default_xxx_filter.xxx_terms))] + } + fake_state = Mock(return_value=Mock(get_progress=Mock(return_value=0.5))) + metadata_store.get_entries = Mock(return_value=[Mock(to_simple_dict=Mock(return_value=fake_entry.copy()))]) + endpoint.tag_rules_processor = Mock(process_queue=AsyncMock()) + endpoint.download_manager.get_download = Mock(return_value=Mock(get_state=fake_state)) + response = await do_request(rest_api, "metadata/torrents/popular", params={"hide_xxx": 1}) + + endpoint.tag_rules_processor.process_queue.assert_called_once() + fake_entry["statements"] = [] # Should be stripped + assert response == {'results': [{**fake_entry, **{"progress": 0.5}}], 'first': 1, 'last': 50} + + +async def test_get_popular_torrents_no_db(rest_api, endpoint, metadata_store): + """ + Test that the endpoint responds with its known entries with statements intact, if no db is present. + """ + fake_entry = { + "name": "Torrent Name", + "category": "", + "infohash": "ab" * 20, + "size": 1, + "num_seeders": 1234, + "num_leechers": 123, + "last_tracker_check": 17000000, + "created": 15000000, + "tag_processor_version": 1, + "type": REGULAR_TORRENT, + "id": 0, + "origin_id": 0, + "public_key": "ab" * 64, + "status": 2, + "statements": [next(iter(default_xxx_filter.xxx_terms))] + } + fake_state = Mock(return_value=Mock(get_progress=Mock(return_value=0.5))) + metadata_store.get_entries = Mock(return_value=[Mock(to_simple_dict=Mock(return_value=fake_entry.copy()))]) + endpoint.tag_rules_processor = Mock(process_queue=AsyncMock()) + endpoint.download_manager.get_download = Mock(return_value=Mock(get_state=fake_state)) + endpoint.tribler_db = None + response = await do_request(rest_api, "metadata/torrents/popular") + + endpoint.tag_rules_processor.process_queue.assert_called_once() + assert response == {'results': [{**fake_entry, **{"progress": 0.5}}], 'first': 1, 'last': 50} + + +async def test_search(rest_api): + """ + Test a search query that should return a few new type channels + """ + + parsed = await do_request(rest_api, 'metadata/search/local?txt_filter=needle', expected_code=200) + assert len(parsed["results"]) == 1 + + parsed = await do_request(rest_api, 'metadata/search/local?txt_filter=hay', expected_code=200) + assert len(parsed["results"]) == 50 + + parsed = await do_request(rest_api, 'metadata/search/local?txt_filter=needle&type=torrent', expected_code=200) + assert parsed["results"][0]['name'] == 'needle' + + parsed = await do_request(rest_api, 'metadata/search/local?txt_filter=needle&sort_by=name', expected_code=200) + assert len(parsed["results"]) == 1 + + parsed = await do_request(rest_api, 'metadata/search/local?txt_filter=needle%2A&sort_by=name&sort_desc=1', + expected_code=200) + assert len(parsed["results"]) == 2 + assert parsed["results"][0]['name'] == "needle2" + + +async def test_search_by_tags(rest_api): + def mocked_get_subjects_intersection(*_, objects: Set[str], **__): + if objects.pop() == 'missed_tag': + return None + return {hexlify(os.urandom(20))} + + with patch.object(KnowledgeDataAccessLayer, 'get_subjects_intersection', wraps=mocked_get_subjects_intersection): + parsed = await do_request(rest_api, 'metadata/search/local?txt_filter=needle&tags=real_tag', expected_code=200) + + assert len(parsed["results"]) == 0 + + parsed = await do_request(rest_api, 'metadata/search/local?txt_filter=needle&tags=missed_tag', + expected_code=200) + assert len(parsed["results"]) == 1 + + +async def test_search_with_include_total_and_max_rowid(rest_api): + """ + Test search queries with include_total and max_rowid options + """ + + parsed = await do_request(rest_api, 'metadata/search/local?txt_filter=needle', expected_code=200) + assert len(parsed["results"]) == 1 + assert "total" not in parsed + assert "max_rowid" not in parsed + + parsed = await do_request(rest_api, 'metadata/search/local?txt_filter=needle&include_total=1', expected_code=200) + assert parsed["total"] == 1 + assert parsed["max_rowid"] == 102 + + parsed = await do_request(rest_api, 'metadata/search/local?txt_filter=hay&include_total=1', expected_code=200) + assert parsed["total"] == 100 + assert parsed["max_rowid"] == 102 + + parsed = await do_request(rest_api, 'metadata/search/local?txt_filter=hay', expected_code=200) + assert len(parsed["results"]) == 50 + + parsed = await do_request(rest_api, 'metadata/search/local?txt_filter=hay&max_rowid=0', expected_code=200) + assert len(parsed["results"]) == 0 + + parsed = await do_request(rest_api, 'metadata/search/local?txt_filter=hay&max_rowid=19', expected_code=200) + assert len(parsed["results"]) == 19 + + parsed = await do_request(rest_api, 'metadata/search/local?txt_filter=needle&sort_by=name', expected_code=200) + assert len(parsed["results"]) == 1 + + parsed = await do_request(rest_api, 'metadata/search/local?txt_filter=needle&sort_by=name&max_rowid=20', + expected_code=200) + assert len(parsed["results"]) == 0 + + parsed = await do_request(rest_api, 'metadata/search/local?txt_filter=needle&sort_by=name&max_rowid=200', + expected_code=200) + assert len(parsed["results"]) == 1 + + +async def test_completions_no_query(rest_api): + """ + Testing whether the API returns an error 400 if no query is passed when getting search completion terms + """ + await do_request(rest_api, 'metadata/search/completions', expected_code=400) + + +async def test_completions(rest_api): + """ + Testing whether the API returns the right terms when getting search completion terms + """ + json_response = await do_request(rest_api, 'metadata/search/completions?q=tribler', expected_code=200) + assert json_response['completions'] == [] + + +async def test_search_with_space(rest_api, metadata_store): + with db_session: + metadata_store.TorrentMetadata(title='abc', infohash=random_infohash(), public_key=b'') + metadata_store.TorrentMetadata(title='abc.def', infohash=random_infohash(), public_key=b'') + metadata_store.TorrentMetadata(title='abc def', infohash=random_infohash(), public_key=b'') + metadata_store.TorrentMetadata(title='abcxyz def', infohash=random_infohash(), public_key=b'') + metadata_store.TorrentMetadata(title='abc defxyz', infohash=random_infohash(), public_key=b'') + + s1 = to_fts_query("abc") + assert s1 == '"abc"' + + s2 = to_fts_query("abc def") + assert s2 == '"abc" "def"' + + ss2 = to_fts_query(s2) + assert ss2 == s2 + + parsed = await do_request(rest_api, f'metadata/search/local?txt_filter={s1}', expected_code=200) + results = {item["name"] for item in parsed["results"]} + assert results == {'abc', 'abc.def', 'abc def', 'abc defxyz'} + + parsed = await do_request(rest_api, f'metadata/search/local?txt_filter={s2}', expected_code=200) + results = {item["name"] for item in parsed["results"]} + assert results == {'abc.def', 'abc def'} # but not 'abcxyz def' + + +async def test_single_snippet_in_search(rest_api, metadata_store): + """ + Test building a simple snippet of a single item. + """ + with db_session: + content_ih = random_infohash() + metadata_store.TorrentMetadata(title='abc', infohash=content_ih) + + def mocked_get_subjects(*_, **__) -> List[str]: + return ["Abc"] + + with patch.object(KnowledgeDataAccessLayer, 'get_objects', wraps=mocked_get_subjects): + s1 = to_fts_query("abc") + results = await do_request(rest_api, f'metadata/search/local?txt_filter={s1}', expected_code=200) + + assert len(results["results"]) == 1 + snippet = results["results"][0] + assert snippet["type"] == SNIPPET + assert snippet["torrents"] == 1 + assert len(snippet["torrents_in_snippet"]) == 1 + assert snippet["torrents_in_snippet"][0]["infohash"] == hexlify(content_ih) + + +async def test_multiple_snippets_in_search(rest_api, metadata_store): + """ + Test two snippets with two torrents in each snippet. + """ + with db_session: + infohashes = [random_infohash() for _ in range(5)] + for ind, infohash in enumerate(infohashes): + torrent_state = metadata_store.TorrentState(infohash=infohash, seeders=ind) + metadata_store.TorrentMetadata(title=f'abc {ind}', infohash=infohash, health=torrent_state, public_key=b'') + + def mocked_get_objects(*__, subject=None, **___) -> List[str]: + subject = unhexlify(subject) + if subject in {infohashes[0], infohashes[1]}: + return ["Content item 1"] + if subject in {infohashes[2], infohashes[3]}: + return ["Content item 2"] + return [] + + with patch.object(KnowledgeDataAccessLayer, 'get_objects', wraps=mocked_get_objects): + s1 = to_fts_query("abc") + parsed = await do_request(rest_api, f'metadata/search/local?txt_filter={s1}', expected_code=200) + results = parsed["results"] + + assert len(results) == 3 + for snippet in results[:2]: + assert snippet["type"] == SNIPPET + assert snippet["torrents"] == 2 + + # Test that the right torrents have been assigned to the appropriate content items, and that they are in the + # right sorted order. + assert results[0]["torrents_in_snippet"][0]["infohash"] == hexlify(infohashes[3]) + assert results[0]["torrents_in_snippet"][1]["infohash"] == hexlify(infohashes[2]) + assert results[1]["torrents_in_snippet"][0]["infohash"] == hexlify(infohashes[1]) + assert results[1]["torrents_in_snippet"][1]["infohash"] == hexlify(infohashes[0]) + + # There is one item that has not been assigned to the snippet. + assert results[2]["type"] == REGULAR_TORRENT + assert results[2]["infohash"] == hexlify(infohashes[4]) + + +def test_build_snippets_no_infohash(endpoint: DatabaseEndpoint): + """ Test building snippets without infohash. The `build_snippets` should return the same results.""" + search_results = [{'dictionary': 'without infohash'}] + result = endpoint.build_snippets(search_results) + assert result == search_results diff --git a/src/tribler/core/components/metadata_store/settings.py b/src/tribler/core/components/database/settings.py similarity index 100% rename from src/tribler/core/components/metadata_store/settings.py rename to src/tribler/core/components/database/settings.py diff --git a/src/tribler/core/components/metadata_store/tests/__init__.py b/src/tribler/core/components/database/tests/__init__.py similarity index 100% rename from src/tribler/core/components/metadata_store/tests/__init__.py rename to src/tribler/core/components/database/tests/__init__.py diff --git a/src/tribler/core/components/metadata_store/tests/test_channel_metadata.py b/src/tribler/core/components/database/tests/test_channel_metadata.py similarity index 94% rename from src/tribler/core/components/metadata_store/tests/test_channel_metadata.py rename to src/tribler/core/components/database/tests/test_channel_metadata.py index 3f6cfc5ca87..a236722557b 100644 --- a/src/tribler/core/components/metadata_store/tests/test_channel_metadata.py +++ b/src/tribler/core/components/database/tests/test_channel_metadata.py @@ -1,4 +1,4 @@ -from tribler.core.components.metadata_store.db.store import HealthItemsPayload +from tribler.core.components.database.db.store import HealthItemsPayload def test_unpack_health_items(): diff --git a/src/tribler/core/components/metadata_store/tests/test_metadata_store_component.py b/src/tribler/core/components/database/tests/test_database_component.py similarity index 64% rename from src/tribler/core/components/metadata_store/tests/test_metadata_store_component.py rename to src/tribler/core/components/database/tests/test_database_component.py index 3aaae2bc544..3ffd7ce9d59 100644 --- a/src/tribler/core/components/metadata_store/tests/test_metadata_store_component.py +++ b/src/tribler/core/components/database/tests/test_database_component.py @@ -2,16 +2,12 @@ from tribler.core.components.ipv8.ipv8_component import Ipv8Component from tribler.core.components.key.key_component import KeyComponent from tribler.core.components.knowledge.knowledge_component import KnowledgeComponent -from tribler.core.components.metadata_store.metadata_store_component import MetadataStoreComponent from tribler.core.components.session import Session -# pylint: disable=protected-access - - -async def test_metadata_store_component(tribler_config): - components = [DatabaseComponent(), KnowledgeComponent(), Ipv8Component(), KeyComponent(), MetadataStoreComponent()] +async def test_database_component(tribler_config): + components = [DatabaseComponent(), KnowledgeComponent(), Ipv8Component(), KeyComponent()] async with Session(tribler_config, components) as session: - comp = session.get_instance(MetadataStoreComponent) + comp = session.get_instance(DatabaseComponent) assert comp.started_event.is_set() and not comp.failed assert comp.mds diff --git a/src/tribler/core/components/metadata_store/tests/test_metadata.py b/src/tribler/core/components/database/tests/test_metadata.py similarity index 100% rename from src/tribler/core/components/metadata_store/tests/test_metadata.py rename to src/tribler/core/components/database/tests/test_metadata.py diff --git a/src/tribler/core/components/metadata_store/tests/test_timeutils.py b/src/tribler/core/components/database/tests/test_timeutils.py similarity index 89% rename from src/tribler/core/components/metadata_store/tests/test_timeutils.py rename to src/tribler/core/components/database/tests/test_timeutils.py index f255c9940c8..2f2acc05734 100644 --- a/src/tribler/core/components/metadata_store/tests/test_timeutils.py +++ b/src/tribler/core/components/database/tests/test_timeutils.py @@ -1,6 +1,6 @@ import datetime -from tribler.core.components.metadata_store.db.serialization import EPOCH, int2time, time2int +from tribler.core.components.database.db.serialization import EPOCH, int2time, time2int def test_time_convert(): diff --git a/src/tribler/core/components/gui_process_watcher/__init__.py b/src/tribler/core/components/gui_process_watcher/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/tribler/core/components/gui_process_watcher/tests/__init__.py b/src/tribler/core/components/gui_process_watcher/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/tribler/core/components/knowledge/knowledge_component.py b/src/tribler/core/components/knowledge/knowledge_component.py index 10830b43fee..b024555d37e 100644 --- a/src/tribler/core/components/knowledge/knowledge_component.py +++ b/src/tribler/core/components/knowledge/knowledge_component.py @@ -1,4 +1,3 @@ -import tribler.core.components.metadata_store.metadata_store_component as metadata_store_component from tribler.core.components.component import Component from tribler.core.components.database.database_component import DatabaseComponent from tribler.core.components.ipv8.ipv8_component import Ipv8Component @@ -19,7 +18,6 @@ async def run(self): self._ipv8_component = await self.require_component(Ipv8Component) key_component = await self.require_component(KeyComponent) - mds_component = await self.require_component(metadata_store_component.MetadataStoreComponent) db_component = await self.require_component(DatabaseComponent) self.community = KnowledgeCommunity( @@ -32,7 +30,7 @@ async def run(self): self.rules_processor = KnowledgeRulesProcessor( notifier=self.session.notifier, db=db_component.db, - mds=mds_component.mds, + mds=db_component.mds, ) self.rules_processor.start() diff --git a/src/tribler/core/components/knowledge/rules/knowledge_rules_processor.py b/src/tribler/core/components/knowledge/rules/knowledge_rules_processor.py index 7d27978e20c..b1e67a379ac 100644 --- a/src/tribler/core/components/knowledge/rules/knowledge_rules_processor.py +++ b/src/tribler/core/components/knowledge/rules/knowledge_rules_processor.py @@ -15,8 +15,8 @@ from tribler.core.components.knowledge.rules.rules_content_items import content_items_rules from tribler.core.components.knowledge.rules.rules_general_tags import general_rules from tribler.core.components.knowledge.rules.tag_rules_base import extract_only_valid_tags -from tribler.core.components.metadata_store.db.serialization import REGULAR_TORRENT -from tribler.core.components.metadata_store.db.store import MetadataStore +from tribler.core.components.database.db.serialization import REGULAR_TORRENT +from tribler.core.components.database.db.store import MetadataStore from tribler.core.utilities.async_force_switch import force_switch from tribler.core.utilities.notifier import Notifier from tribler.core.utilities.unicode import hexlify diff --git a/src/tribler/core/components/knowledge/rules/tests/test_knowledge_rules_processor.py b/src/tribler/core/components/knowledge/rules/tests/test_knowledge_rules_processor.py index ba00ffa774a..0605cae013c 100644 --- a/src/tribler/core/components/knowledge/rules/tests/test_knowledge_rules_processor.py +++ b/src/tribler/core/components/knowledge/rules/tests/test_knowledge_rules_processor.py @@ -8,8 +8,8 @@ from tribler.core.components.database.db.layers.knowledge_data_access_layer import ResourceType from tribler.core.components.database.db.tribler_database import TriblerDatabase from tribler.core.components.knowledge.rules.knowledge_rules_processor import KnowledgeRulesProcessor -from tribler.core.components.metadata_store.db.serialization import REGULAR_TORRENT -from tribler.core.components.metadata_store.db.store import MetadataStore +from tribler.core.components.database.db.serialization import REGULAR_TORRENT +from tribler.core.components.database.db.store import MetadataStore from tribler.core.utilities.path_util import Path from tribler.core.utilities.utilities import MEMORY_DB diff --git a/src/tribler/core/components/knowledge/tests/test_knowledge_component.py b/src/tribler/core/components/knowledge/tests/test_knowledge_component.py index d0568b100ac..771eb8000fd 100644 --- a/src/tribler/core/components/knowledge/tests/test_knowledge_component.py +++ b/src/tribler/core/components/knowledge/tests/test_knowledge_component.py @@ -2,7 +2,6 @@ from tribler.core.components.ipv8.ipv8_component import Ipv8Component from tribler.core.components.key.key_component import KeyComponent from tribler.core.components.knowledge.knowledge_component import KnowledgeComponent -from tribler.core.components.metadata_store.metadata_store_component import MetadataStoreComponent from tribler.core.components.session import Session @@ -10,7 +9,7 @@ async def test_tag_component(tribler_config): - components = [DatabaseComponent(), MetadataStoreComponent(), KeyComponent(), Ipv8Component(), KnowledgeComponent()] + components = [DatabaseComponent(), KeyComponent(), Ipv8Component(), KnowledgeComponent()] async with Session(tribler_config, components) as session: comp = session.get_instance(KnowledgeComponent) assert comp.started_event.is_set() and not comp.failed diff --git a/src/tribler/core/components/libtorrent/restapi/tests/test_torrentinfo_endpoint.py b/src/tribler/core/components/libtorrent/restapi/tests/test_torrentinfo_endpoint.py index c3a46725c00..f3434054e98 100644 --- a/src/tribler/core/components/libtorrent/restapi/tests/test_torrentinfo_endpoint.py +++ b/src/tribler/core/components/libtorrent/restapi/tests/test_torrentinfo_endpoint.py @@ -15,7 +15,7 @@ from tribler.core.components.libtorrent.restapi.torrentinfo_endpoint import TorrentInfoEndpoint from tribler.core.components.libtorrent.settings import DownloadDefaultsSettings, LibtorrentSettings from tribler.core.components.libtorrent.torrentdef import TorrentDef -from tribler.core.components.metadata_store.db.orm_bindings.torrent_metadata import tdef_to_metadata_dict +from tribler.core.components.database.db.orm_bindings.torrent_metadata import tdef_to_metadata_dict from tribler.core.components.restapi.rest.base_api_test import do_request from tribler.core.components.restapi.rest.rest_endpoint import HTTP_INTERNAL_SERVER_ERROR from tribler.core.tests.tools.common import TESTS_DATA_DIR, TESTS_DIR, TORRENT_UBUNTU_FILE, UBUNTU_1504_INFOHASH diff --git a/src/tribler/core/components/libtorrent/restapi/torrentinfo_endpoint.py b/src/tribler/core/components/libtorrent/restapi/torrentinfo_endpoint.py index 53a8859b7bf..0465ec41172 100644 --- a/src/tribler/core/components/libtorrent/restapi/torrentinfo_endpoint.py +++ b/src/tribler/core/components/libtorrent/restapi/torrentinfo_endpoint.py @@ -13,7 +13,7 @@ from tribler.core.components.libtorrent.download_manager.download_manager import DownloadManager from tribler.core.components.libtorrent.torrentdef import TorrentDef from tribler.core.components.libtorrent.utils.libtorrent_helper import libtorrent as lt -from tribler.core.components.metadata_store.db.orm_bindings.torrent_metadata import tdef_to_metadata_dict +from tribler.core.components.database.db.orm_bindings.torrent_metadata import tdef_to_metadata_dict from tribler.core.components.restapi.rest.rest_endpoint import ( HTTP_BAD_REQUEST, HTTP_INTERNAL_SERVER_ERROR, diff --git a/src/tribler/core/components/metadata_store/metadata_store_component.py b/src/tribler/core/components/metadata_store/metadata_store_component.py deleted file mode 100644 index 4c16e328c6e..00000000000 --- a/src/tribler/core/components/metadata_store/metadata_store_component.py +++ /dev/null @@ -1,53 +0,0 @@ -from tribler.core import notifications -from tribler.core.components.component import Component -from tribler.core.components.key.key_component import KeyComponent -from tribler.core.components.knowledge.rules.knowledge_rules_processor import KnowledgeRulesProcessor -from tribler.core.components.metadata_store.db.store import MetadataStore -from tribler.core.utilities.simpledefs import STATEDIR_DB_DIR - - -class MetadataStoreComponent(Component): - mds: MetadataStore = None - - async def run(self): - await super().run() - - config = self.session.config - channels_dir = config.chant.get_path_as_absolute('channels_dir', config.state_dir) - chant_testnet = config.general.testnet or config.chant.testnet - - metadata_db_name = 'metadata.db' - if chant_testnet: - metadata_db_name = 'metadata_testnet.db' - elif config.gui_test_mode: # Avoid interfering with the main database in test mode - # Note we don't use in-memory database in core test mode, because MDS uses threads, - # and SQLite creates a different in-memory DB for each connection by default. - # To change this behaviour, we have to use url-based SQLite initialization syntax, - # which is not supported by PonyORM yet. - metadata_db_name = 'metadata_gui_test.db' - - database_path = config.state_dir / STATEDIR_DB_DIR / metadata_db_name - - # Make sure that we start with a clean metadata database when in GUI mode every time. - if config.gui_test_mode and database_path.exists(): - self.logger.info("Wiping metadata database in GUI test mode") - database_path.unlink(missing_ok=True) - - key_component = await self.require_component(KeyComponent) - - metadata_store = MetadataStore( - database_path, - channels_dir, - key_component.primary_key, - notifier=self.session.notifier, - disable_sync=config.gui_test_mode, - tag_processor_version=KnowledgeRulesProcessor.version - ) - self.mds = metadata_store - self.session.notifier.add_observer(notifications.torrent_metadata_added, - metadata_store.TorrentMetadata.add_ffa_from_dict) - - async def shutdown(self): - await super().shutdown() - if self.mds: - self.mds.shutdown() diff --git a/src/tribler/core/components/metadata_store/restapi/metadata_endpoint.py b/src/tribler/core/components/metadata_store/restapi/metadata_endpoint.py deleted file mode 100644 index 0b1bb35a473..00000000000 --- a/src/tribler/core/components/metadata_store/restapi/metadata_endpoint.py +++ /dev/null @@ -1,139 +0,0 @@ -from binascii import unhexlify -from typing import Optional - -from aiohttp import web -from aiohttp_apispec import docs -from ipv8.REST.base_endpoint import HTTP_BAD_REQUEST -from ipv8.REST.schema import schema -from marshmallow.fields import Boolean, Integer -from pony.orm import db_session - -from tribler.core.components.libtorrent.download_manager.download_manager import DownloadManager -from tribler.core.components.metadata_store.db.serialization import REGULAR_TORRENT -from tribler.core.components.metadata_store.restapi.metadata_endpoint_base import MetadataEndpointBase -from tribler.core.components.metadata_store.restapi.metadata_schema import TorrentSchema -from tribler.core.components.restapi.rest.rest_endpoint import RESTResponse -from tribler.core.components.torrent_checker.torrent_checker.torrent_checker import TorrentChecker -from tribler.core.utilities.utilities import froze_it - -TORRENT_CHECK_TIMEOUT = 20 - - -@froze_it -class MetadataEndpoint(MetadataEndpointBase): - """ - This is the top-level endpoint class that serves other endpoints. - - # /metadata - # /torrents - # / - """ - path = '/metadata' - - def __init__(self, download_manager: DownloadManager, - torrent_checker: Optional[TorrentChecker], *args, **kwargs): - MetadataEndpointBase.__init__(self, *args, **kwargs) - self.download_manager = download_manager - self.torrent_checker = torrent_checker - - def setup_routes(self): - self.app.add_routes( - [ - web.get('/torrents/{infohash}/health', self.get_torrent_health), - web.get('/torrents/popular', self.get_popular_torrents), - ] - ) - - @docs( - tags=["Metadata"], - summary="Fetch the swarm health of a specific torrent.", - parameters=[ - { - 'in': 'path', - 'name': 'infohash', - 'description': 'Infohash of the download to remove', - 'type': 'string', - 'required': True, - }, - { - 'in': 'query', - 'name': 'timeout', - 'description': 'Timeout to be used in the connections to the trackers', - 'type': 'integer', - 'default': 20, - 'required': False, - }, - ], - responses={ - 200: { - 'schema': schema( - HealthCheckResponse={ - 'checking': Boolean() - } - ), - 'examples': [ - {'checking': 1}, - ], - } - }, - ) - async def get_torrent_health(self, request): - self._logger.info(f'Get torrent health request: {request}') - try: - timeout = int(request.query.get('timeout', TORRENT_CHECK_TIMEOUT)) - except ValueError as e: - return RESTResponse({"error": f"Error processing timeout parameter: {e}"}, status=HTTP_BAD_REQUEST) - - if self.torrent_checker is None: - return RESTResponse({'checking': False}) - - infohash = unhexlify(request.match_info['infohash']) - check_coro = self.torrent_checker.check_torrent_health(infohash, timeout=timeout, scrape_now=True) - self.async_group.add_task(check_coro) - return RESTResponse({'checking': True}) - - def add_download_progress_to_metadata_list(self, contents_list): - for torrent in contents_list: - if torrent['type'] == REGULAR_TORRENT: - dl = self.download_manager.get_download(unhexlify(torrent['infohash'])) - if dl is not None and dl.tdef.infohash not in self.download_manager.metainfo_requests: - torrent['progress'] = dl.get_state().get_progress() - - @docs( - tags=['Metadata'], - summary='Get the list of most popular torrents.', - responses={ - 200: { - 'schema': schema( - GetPopularTorrentsResponse={ - 'results': [TorrentSchema], - 'first': Integer(), - 'last': Integer(), - } - ) - } - }, - ) - async def get_popular_torrents(self, request): - sanitized = self.sanitize_parameters(request.query) - sanitized["metadata_type"] = REGULAR_TORRENT - sanitized["popular"] = True - - with db_session: - contents = self.mds.get_entries(**sanitized) - contents_list = [] - for entry in contents: - contents_list.append(entry.to_simple_dict()) - - if self.tag_rules_processor: - await self.tag_rules_processor.process_queue() - - self.add_download_progress_to_metadata_list(contents_list) - self.add_statements_to_metadata_list(contents_list, hide_xxx=sanitized["hide_xxx"]) - response_dict = { - "results": contents_list, - "first": sanitized['first'], - "last": sanitized['last'], - } - - return RESTResponse(response_dict) diff --git a/src/tribler/core/components/metadata_store/restapi/metadata_endpoint_base.py b/src/tribler/core/components/metadata_store/restapi/metadata_endpoint_base.py deleted file mode 100644 index 541452d20ac..00000000000 --- a/src/tribler/core/components/metadata_store/restapi/metadata_endpoint_base.py +++ /dev/null @@ -1,93 +0,0 @@ -from dataclasses import asdict -from typing import Optional - -from pony.orm import db_session - -from tribler.core.components.database.db.layers.knowledge_data_access_layer import ResourceType -from tribler.core.components.database.db.tribler_database import TriblerDatabase -from tribler.core.components.knowledge.rules.knowledge_rules_processor import KnowledgeRulesProcessor -from tribler.core.components.metadata_store.category_filter.family_filter import default_xxx_filter -from tribler.core.components.metadata_store.db.serialization import CHANNEL_TORRENT, COLLECTION_NODE, REGULAR_TORRENT -from tribler.core.components.metadata_store.db.store import MetadataStore -from tribler.core.components.restapi.rest.rest_endpoint import RESTEndpoint - -from tribler.core.utilities.utilities import parse_bool - -# This dict is used to translate JSON fields into the columns used in Pony for _sorting_. -# id_ is not in the list because there is not index on it, so we never really want to sort on it. - -json2pony_columns = { - 'category': "tags", - 'name': "title", - 'size': "size", - 'infohash': "infohash", - 'date': "torrent_date", - 'created': "torrent_date", - 'status': 'status', - 'torrents': 'num_entries', - 'votes': 'votes', - 'subscribed': 'subscribed', - 'health': 'HEALTH', -} - -# TODO: use the same representation for metadata nodes as in the database -metadata_type_to_search_scope = { - '': frozenset((REGULAR_TORRENT, CHANNEL_TORRENT, COLLECTION_NODE)), - "channel": frozenset((CHANNEL_TORRENT, COLLECTION_NODE)), - "torrent": frozenset((REGULAR_TORRENT,)), - str(CHANNEL_TORRENT): frozenset((CHANNEL_TORRENT,)), - str(REGULAR_TORRENT): frozenset((REGULAR_TORRENT,)), - str(COLLECTION_NODE): frozenset((COLLECTION_NODE,)), -} - - -class MetadataEndpointBase(RESTEndpoint): - def __init__(self, metadata_store: MetadataStore, *args, tribler_db: TriblerDatabase = None, - tag_rules_processor: KnowledgeRulesProcessor = None, **kwargs): - super().__init__(*args, **kwargs) - self.mds = metadata_store - self.tribler_db: Optional[TriblerDatabase] = tribler_db - self.tag_rules_processor: Optional[KnowledgeRulesProcessor] = tag_rules_processor - - @classmethod - def sanitize_parameters(cls, parameters): - """ - Sanitize the parameters for a request that fetches channels. - """ - sanitized = { - "first": int(parameters.get('first', 1)), - "last": int(parameters.get('last', 50)), - "sort_by": json2pony_columns.get(parameters.get('sort_by')), - "sort_desc": parse_bool(parameters.get('sort_desc', True)), - "txt_filter": parameters.get('txt_filter'), - "hide_xxx": parse_bool(parameters.get('hide_xxx', False)), - "category": parameters.get('category'), - } - if 'tags' in parameters: - sanitized['tags'] = parameters.getall('tags') - if "remote" in parameters: - sanitized["remote"] = (parse_bool(parameters.get('remote', False)),) - if 'metadata_type' in parameters: - mtypes = [] - for arg in parameters.getall('metadata_type'): - mtypes.extend(metadata_type_to_search_scope[arg]) - sanitized['metadata_type'] = frozenset(mtypes) - return sanitized - - @db_session - def add_statements_to_metadata_list(self, contents_list, hide_xxx=False): - if self.tribler_db is None: - self._logger.error(f'Cannot add statements to metadata list: ' - f'tribler_db is not set in {self.__class__.__name__}') - return - for torrent in contents_list: - if torrent['type'] == REGULAR_TORRENT: - raw_statements = self.tribler_db.knowledge.get_statements( - subject_type=ResourceType.TORRENT, - subject=torrent["infohash"] - ) - statements = [asdict(stmt) for stmt in raw_statements] - if hide_xxx: - statements = [stmt for stmt in statements if not default_xxx_filter.isXXX(stmt["object"], - isFilename=False)] - torrent["statements"] = statements diff --git a/src/tribler/core/components/metadata_store/restapi/tests/conftest.py b/src/tribler/core/components/metadata_store/restapi/tests/conftest.py deleted file mode 100644 index 4ba11ff2c6a..00000000000 --- a/src/tribler/core/components/metadata_store/restapi/tests/conftest.py +++ /dev/null @@ -1,84 +0,0 @@ -import hashlib -from time import time - -import pytest -from ipv8.keyvault.crypto import default_eccrypto -from pony.orm import db_session - -from tribler.core.components.metadata_store.db.orm_bindings.torrent_metadata import NEW -from tribler.core.components.metadata_store.utils import tag_torrent -from tribler.core.utilities.utilities import random_infohash - - -@db_session -def create_channel(metadata_store, title, torrents_per_channel=5, local_version=0, subscribed=True): - def sha1_hash(value: str) -> bytes: - return hashlib.sha1(value.encode('utf-8')).digest() - - def add_torrent_to_channel(torrent_title, seeders, self_checked): - t = metadata_store.TorrentMetadata(origin_id=channel.id_, title=torrent_title, - infohash=sha1_hash(torrent_title), sign_with=key) - t.health.seeders = seeders - t.health.self_checked = self_checked - - one_week = 60 * 60 * 24 * 7 - now = int(time()) - - t.health.last_check = now - one_week if self_checked else now - - key = default_eccrypto.generate_key('curve25519') - - channel = metadata_store.ChannelMetadata(title=title, subscribed=subscribed, - num_entries=torrents_per_channel, - infohash=sha1_hash(title), id_=123, - sign_with=key, version=10, - local_version=local_version) - - for torrent_i in range(torrents_per_channel): - add_torrent_to_channel(f'torrent{channel.title}{torrent_i}', seeders=torrent_i, - self_checked=bool(torrent_i % 2)) - - -@pytest.fixture -def add_subscribed_and_not_downloaded_channel(metadata_store): - create_channel(metadata_store, 'Subscribed and not downloaded', subscribed=True, local_version=0) - - -@pytest.fixture -def add_fake_torrents_channels(metadata_store): - for i in range(10): - create_channel(metadata_store, f'channel{i}', subscribed=bool(i % 2), local_version=i) - - -@pytest.fixture -def my_channel(metadata_store, tribler_db): - """ - Generate a channel with some torrents. Also add a few (random) tags to these torrents. - """ - with db_session: - chan = metadata_store.ChannelMetadata.create_channel('test', 'test') - for ind in range(5): - infohash = random_infohash() - _ = metadata_store.TorrentMetadata( - origin_id=chan.id_, title='torrent%d' % ind, status=NEW, infohash=infohash - ) - tag_torrent(infohash, tribler_db) - for ind in range(5, 9): - infohash = random_infohash() - _ = metadata_store.TorrentMetadata(origin_id=chan.id_, title='torrent%d' % ind, infohash=infohash) - tag_torrent(infohash, tribler_db) - - chan2 = metadata_store.ChannelMetadata.create_channel('test2', 'test2') - for ind in range(5): - infohash = random_infohash() - _ = metadata_store.TorrentMetadata( - origin_id=chan2.id_, title='torrentB%d' % ind, status=NEW, infohash=infohash - ) - tag_torrent(infohash, tribler_db) - for ind in range(5, 9): - infohash = random_infohash() - _ = metadata_store.TorrentMetadata( - origin_id=chan2.id_, title='torrentB%d' % ind, infohash=random_infohash() - ) - tag_torrent(infohash, tribler_db) - return chan diff --git a/src/tribler/core/components/metadata_store/restapi/tests/test_metadata_endpoint.py b/src/tribler/core/components/metadata_store/restapi/tests/test_metadata_endpoint.py deleted file mode 100644 index 8d742fa217b..00000000000 --- a/src/tribler/core/components/metadata_store/restapi/tests/test_metadata_endpoint.py +++ /dev/null @@ -1,86 +0,0 @@ -from unittest.mock import MagicMock, Mock, AsyncMock - -import pytest - -from tribler.core.components.metadata_store.db.serialization import REGULAR_TORRENT -from tribler.core.components.metadata_store.restapi.metadata_endpoint import MetadataEndpoint, TORRENT_CHECK_TIMEOUT -from tribler.core.components.restapi.rest.base_api_test import do_request -from tribler.core.components.torrent_checker.torrent_checker.torrent_checker import TorrentChecker -from tribler.core.config.tribler_config import TriblerConfig -from tribler.core.utilities.unicode import hexlify - - -# pylint: disable=unused-argument, redefined-outer-name - -@pytest.fixture -async def torrent_checker(mock_dlmgr, metadata_store): - # Initialize the torrent checker - config = TriblerConfig() - config.download_defaults.number_hops = 0 - tracker_manager = MagicMock() - tracker_manager.blacklist = [] - notifier = MagicMock() - torrent_checker = TorrentChecker( - config=config, - download_manager=mock_dlmgr, - tracker_manager=tracker_manager, - metadata_store=metadata_store, - notifier=notifier, - socks_listen_ports=[2000, 3000], - ) - await torrent_checker.initialize() - yield torrent_checker - await torrent_checker.shutdown() - - -@pytest.fixture -def endpoint(torrent_checker, metadata_store): - return MetadataEndpoint(torrent_checker.download_manager, torrent_checker, metadata_store) - - -async def test_check_torrent_health(rest_api, mock_dlmgr, udp_tracker, metadata_store): - """ - Test the endpoint to fetch the health of a chant-managed, infohash-only torrent - """ - infohash = b'a' * 20 - url = f'metadata/torrents/{hexlify(infohash)}/health?timeout={TORRENT_CHECK_TIMEOUT}' - json_response = await do_request(rest_api, url) - assert json_response == {'checking': True} - - -async def test_check_torrent_query(rest_api, udp_tracker, metadata_store): - """ - Test that the endpoint responds with an error message if the timeout parameter has a wrong value - """ - infohash = b'a' * 20 - await do_request(rest_api, f"metadata/torrents/{infohash}/health?timeout=wrong_value&refresh=1", expected_code=400) - - -async def test_get_popular_torrents(rest_api, endpoint, metadata_store): - """ - Test that the endpoint responds with its known entries. - """ - fake_entry = { - "name": "Torrent Name", - "category": "", - "infohash": "ab" * 20, - "size": 1, - "num_seeders": 1234, - "num_leechers": 123, - "last_tracker_check": 17000000, - "created": 15000000, - "tag_processor_version": 1, - "type": REGULAR_TORRENT, - "id": 0, - "origin_id": 0, - "public_key": "ab" * 64, - "status": 2, - } - fake_state = Mock(return_value=Mock(get_progress=Mock(return_value=0.5))) - metadata_store.get_entries = Mock(return_value=[Mock(to_simple_dict=Mock(return_value=fake_entry.copy()))]) - endpoint.tag_rules_processor = Mock(process_queue=AsyncMock()) - endpoint.download_manager.get_download = Mock(return_value=Mock(get_state=fake_state)) - response = await do_request(rest_api, f"metadata/torrents/popular") - - endpoint.tag_rules_processor.process_queue.assert_called_once() - assert response == {'results': [{**fake_entry, **{"progress": 0.5}}], 'first': 1, 'last': 50} diff --git a/src/tribler/core/components/metadata_store/restapi/tests/test_search_endpoint.py b/src/tribler/core/components/metadata_store/restapi/tests/test_search_endpoint.py deleted file mode 100644 index 5a584d3ae88..00000000000 --- a/src/tribler/core/components/metadata_store/restapi/tests/test_search_endpoint.py +++ /dev/null @@ -1,277 +0,0 @@ -import os -import uuid -from binascii import unhexlify -from typing import List, Set -from unittest.mock import patch, Mock - -import pytest -from pony.orm import db_session - -from tribler.core.components.database.db.layers.knowledge_data_access_layer import KnowledgeDataAccessLayer -from tribler.core.components.metadata_store.db.serialization import REGULAR_TORRENT, SNIPPET -from tribler.core.components.metadata_store.restapi.search_endpoint import SearchEndpoint -from tribler.core.components.restapi.rest.base_api_test import do_request -from tribler.core.utilities.unicode import hexlify -from tribler.core.utilities.utilities import random_infohash, to_fts_query - - -# pylint: disable=unused-argument, redefined-outer-name - - -@pytest.fixture -def needle_in_haystack_mds(metadata_store): - num_hay = 100 - with db_session: - for x in range(0, num_hay): - metadata_store.TorrentMetadata(title='hay ' + str(x), infohash=random_infohash(), public_key=b'') - metadata_store.TorrentMetadata(title='needle', infohash=random_infohash(), public_key=b'') - metadata_store.TorrentMetadata(title='needle2', infohash=random_infohash(), public_key=b'') - return metadata_store - - -@pytest.fixture -def mock_content_discovery_community(): - return Mock() - - -@pytest.fixture -def endpoint(mock_content_discovery_community, needle_in_haystack_mds, tribler_db): - return SearchEndpoint(mock_content_discovery_community, needle_in_haystack_mds, tribler_db=tribler_db) - - -async def test_search_wrong_mdtype(rest_api): - """ - Testing whether the API returns an error 400 if wrong metadata type is passed in the query - """ - await do_request(rest_api, 'search/local?txt_filter=bla&metadata_type=ddd', expected_code=400) - - -async def test_search(rest_api): - """ - Test a search query that should return a few new type channels - """ - - parsed = await do_request(rest_api, 'search/local?txt_filter=needle', expected_code=200) - assert len(parsed["results"]) == 1 - - parsed = await do_request(rest_api, 'search/local?txt_filter=hay', expected_code=200) - assert len(parsed["results"]) == 50 - - parsed = await do_request(rest_api, 'search/local?txt_filter=needle&type=torrent', expected_code=200) - assert parsed["results"][0]['name'] == 'needle' - - parsed = await do_request(rest_api, 'search/local?txt_filter=needle&sort_by=name', expected_code=200) - assert len(parsed["results"]) == 1 - - parsed = await do_request(rest_api, 'search/local?txt_filter=needle%2A&sort_by=name&sort_desc=1', expected_code=200) - assert len(parsed["results"]) == 2 - assert parsed["results"][0]['name'] == "needle2" - - -async def test_search_by_tags(rest_api): - def mocked_get_subjects_intersection(*_, objects: Set[str], **__): - if objects.pop() == 'missed_tag': - return None - return {hexlify(os.urandom(20))} - - with patch.object(KnowledgeDataAccessLayer, 'get_subjects_intersection', wraps=mocked_get_subjects_intersection): - parsed = await do_request(rest_api, 'search/local?txt_filter=needle&tags=real_tag', expected_code=200) - - assert len(parsed["results"]) == 0 - - parsed = await do_request(rest_api, 'search/local?txt_filter=needle&tags=missed_tag', expected_code=200) - assert len(parsed["results"]) == 1 - - -async def test_search_with_include_total_and_max_rowid(rest_api): - """ - Test search queries with include_total and max_rowid options - """ - - parsed = await do_request(rest_api, 'search/local?txt_filter=needle', expected_code=200) - assert len(parsed["results"]) == 1 - assert "total" not in parsed - assert "max_rowid" not in parsed - - parsed = await do_request(rest_api, 'search/local?txt_filter=needle&include_total=1', expected_code=200) - assert parsed["total"] == 1 - assert parsed["max_rowid"] == 102 - - parsed = await do_request(rest_api, 'search/local?txt_filter=hay&include_total=1', expected_code=200) - assert parsed["total"] == 100 - assert parsed["max_rowid"] == 102 - - parsed = await do_request(rest_api, 'search/local?txt_filter=hay', expected_code=200) - assert len(parsed["results"]) == 50 - - parsed = await do_request(rest_api, 'search/local?txt_filter=hay&max_rowid=0', expected_code=200) - assert len(parsed["results"]) == 0 - - parsed = await do_request(rest_api, 'search/local?txt_filter=hay&max_rowid=19', expected_code=200) - assert len(parsed["results"]) == 19 - - parsed = await do_request(rest_api, 'search/local?txt_filter=needle&sort_by=name', expected_code=200) - assert len(parsed["results"]) == 1 - - parsed = await do_request(rest_api, 'search/local?txt_filter=needle&sort_by=name&max_rowid=20', expected_code=200) - assert len(parsed["results"]) == 0 - - parsed = await do_request(rest_api, 'search/local?txt_filter=needle&sort_by=name&max_rowid=200', expected_code=200) - assert len(parsed["results"]) == 1 - - -async def test_completions_no_query(rest_api): - """ - Testing whether the API returns an error 400 if no query is passed when getting search completion terms - """ - await do_request(rest_api, 'search/completions', expected_code=400) - - -async def test_completions(rest_api): - """ - Testing whether the API returns the right terms when getting search completion terms - """ - json_response = await do_request(rest_api, 'search/completions?q=tribler', expected_code=200) - assert json_response['completions'] == [] - - -async def test_search_with_space(rest_api, metadata_store): - with db_session: - metadata_store.TorrentMetadata(title='abc', infohash=random_infohash(), public_key=b'') - metadata_store.TorrentMetadata(title='abc.def', infohash=random_infohash(), public_key=b'') - metadata_store.TorrentMetadata(title='abc def', infohash=random_infohash(), public_key=b'') - metadata_store.TorrentMetadata(title='abcxyz def', infohash=random_infohash(), public_key=b'') - metadata_store.TorrentMetadata(title='abc defxyz', infohash=random_infohash(), public_key=b'') - - s1 = to_fts_query("abc") - assert s1 == '"abc"' - - s2 = to_fts_query("abc def") - assert s2 == '"abc" "def"' - - ss2 = to_fts_query(s2) - assert ss2 == s2 - - parsed = await do_request(rest_api, f'search/local?txt_filter={s1}', expected_code=200) - results = {item["name"] for item in parsed["results"]} - assert results == {'abc', 'abc.def', 'abc def', 'abc defxyz'} - - parsed = await do_request(rest_api, f'search/local?txt_filter={s2}', expected_code=200) - results = {item["name"] for item in parsed["results"]} - assert results == {'abc.def', 'abc def'} # but not 'abcxyz def' - - -async def test_single_snippet_in_search(rest_api, metadata_store, tribler_db): - """ - Test building a simple snippet of a single item. - """ - with db_session: - content_ih = random_infohash() - metadata_store.TorrentMetadata(title='abc', infohash=content_ih) - - def mocked_get_subjects(*_, **__) -> List[str]: - return ["Abc"] - - with patch.object(KnowledgeDataAccessLayer, 'get_objects', wraps=mocked_get_subjects): - s1 = to_fts_query("abc") - results = await do_request(rest_api, f'search/local?txt_filter={s1}', expected_code=200) - - assert len(results["results"]) == 1 - snippet = results["results"][0] - assert snippet["type"] == SNIPPET - assert snippet["torrents"] == 1 - assert len(snippet["torrents_in_snippet"]) == 1 - assert snippet["torrents_in_snippet"][0]["infohash"] == hexlify(content_ih) - - -async def test_multiple_snippets_in_search(rest_api, metadata_store, tribler_db): - """ - Test two snippets with two torrents in each snippet. - """ - with db_session: - infohashes = [random_infohash() for _ in range(5)] - for ind, infohash in enumerate(infohashes): - torrent_state = metadata_store.TorrentState(infohash=infohash, seeders=ind) - metadata_store.TorrentMetadata(title=f'abc {ind}', infohash=infohash, health=torrent_state, public_key=b'') - - def mocked_get_objects(*__, subject=None, **___) -> List[str]: - subject = unhexlify(subject) - if subject in {infohashes[0], infohashes[1]}: - return ["Content item 1"] - if subject in {infohashes[2], infohashes[3]}: - return ["Content item 2"] - return [] - - with patch.object(KnowledgeDataAccessLayer, 'get_objects', wraps=mocked_get_objects): - s1 = to_fts_query("abc") - parsed = await do_request(rest_api, f'search/local?txt_filter={s1}', expected_code=200) - results = parsed["results"] - - assert len(results) == 3 - for snippet in results[:2]: - assert snippet["type"] == SNIPPET - assert snippet["torrents"] == 2 - - # Test that the right torrents have been assigned to the appropriate content items, and that they are in the - # right sorted order. - assert results[0]["torrents_in_snippet"][0]["infohash"] == hexlify(infohashes[3]) - assert results[0]["torrents_in_snippet"][1]["infohash"] == hexlify(infohashes[2]) - assert results[1]["torrents_in_snippet"][0]["infohash"] == hexlify(infohashes[1]) - assert results[1]["torrents_in_snippet"][1]["infohash"] == hexlify(infohashes[0]) - - # There is one item that has not been assigned to the snippet. - assert results[2]["type"] == REGULAR_TORRENT - assert results[2]["infohash"] == hexlify(infohashes[4]) - - -def test_build_snippets_no_infohash(endpoint: SearchEndpoint): - """ Test building snippets without infohash. The `build_snippets` should return the same results.""" - search_results = [{'dictionary': 'without infohash'}] - result = endpoint.build_snippets(search_results) - assert result == search_results - - -async def test_create_remote_search_request(rest_api, mock_content_discovery_community): - """ - Test that remote search call is sent on a REST API search request - """ - sent = {} - peers = [] - request_uuid = uuid.uuid4() - - def mock_send(**kwargs): - sent.update(kwargs) - return request_uuid, peers - - # Test querying for keywords - mock_content_discovery_community.send_search_request = mock_send - search_txt = "foo" - await do_request( - rest_api, - f'search/remote?txt_filter={search_txt}', - request_type="PUT", - expected_code=200, - expected_json={"request_uuid": str(request_uuid), "peers": peers}, - ) - assert sent['txt_filter'] == search_txt - sent.clear() - - # Test querying channel data by public key, e.g. for channel preview purposes - channel_pk = "ff" - await do_request( - rest_api, f'search/remote?channel_pk={channel_pk}&metadata_type=torrent', request_type="PUT", expected_code=200 - ) - assert hexlify(sent['channel_pk']) == channel_pk - - -async def test_create_remote_search_request_illegal(rest_api, mock_content_discovery_community): - """ - Test that remote search call is sent on a REST API search request - """ - response = await do_request( - rest_api, - f'search/remote?origin_id=a', - request_type="PUT", - expected_code=400 - ) - assert "error" in response diff --git a/src/tribler/core/components/metadata_store/utils.py b/src/tribler/core/components/metadata_store/utils.py deleted file mode 100644 index a4a439428c4..00000000000 --- a/src/tribler/core/components/metadata_store/utils.py +++ /dev/null @@ -1,98 +0,0 @@ -import random -import time - -from faker import Faker -from ipv8.keyvault.crypto import default_eccrypto -from pony.orm import db_session - -from tribler.core.components.database.db.layers.knowledge_data_access_layer import Operation, ResourceType -from tribler.core.components.knowledge.community.knowledge_payload import StatementOperation -from tribler.core.components.knowledge.knowledge_constants import MIN_RESOURCE_LENGTH -from tribler.core.utilities.unicode import hexlify -from tribler.core.utilities.utilities import random_infohash - -# Some random keys used for generating tags. -random_key_1 = default_eccrypto.generate_key('low') -random_key_2 = default_eccrypto.generate_key('low') -random_key_3 = default_eccrypto.generate_key('low') -fake = Faker() - - -class RequestTimeoutException(Exception): - pass - - -class NoChannelSourcesException(Exception): - pass - - -def generate_title(words_count=5): - return fake.sentence(nb_words=words_count)[:-1] - - -def get_random_word(min_length=0): - word = fake.word() - while len(word) < min_length: - word = fake.word() - return word - - -def tag_torrent(infohash, db, tags=None, suggested_tags=None): - infohash = hexlify(infohash) - if tags is None: - tags_count = random.randint(2, 6) - tags = [] - while len(tags) < tags_count: - tag = get_random_word(min_length=MIN_RESOURCE_LENGTH) - if tag not in tags: - tags.append(tag) - - if suggested_tags is None: - suggested_tags_count = random.randint(1, 3) - suggested_tags = [] - while len(suggested_tags) < suggested_tags_count: - tag = get_random_word(min_length=MIN_RESOURCE_LENGTH) - if tag not in suggested_tags: - suggested_tags.append(tag) - - def _add_operation(_obj, _op, _key, _predicate=ResourceType.TAG): - operation = StatementOperation(subject_type=ResourceType.TORRENT, subject=infohash, predicate=_predicate, - object=_obj, operation=_op, clock=0, creator_public_key=_key.pub().key_to_bin()) - operation.clock = db.knowledge.get_clock(operation) + 1 - db.knowledge.add_operation(operation, b"") - - # Give the torrent some tags - for tag in tags: - for key in [random_key_1, random_key_2]: # Each tag should be proposed by two unique users - _add_operation(tag, Operation.ADD, key) - - # Make sure we have some suggestions - for tag in suggested_tags: - _add_operation(tag, Operation.ADD, random_key_3) - _add_operation(tag, Operation.REMOVE, random_key_2) - - # Give the torrent some simple attributes - random_title = generate_title(2) - random_year = f"{random.randint(1990, 2040)}" - random_description = generate_title(5) - random_lang = random.choice(["english", "russian", "dutch", "klingon", "valyerian"]) - for key in [random_key_1, random_key_2]: # Each statement should be proposed by two unique users - _add_operation(random_title, Operation.ADD, key, _predicate=ResourceType.TITLE) - _add_operation(random_year, Operation.ADD, key, _predicate=ResourceType.DATE) - _add_operation(random_description, Operation.ADD, key, _predicate=ResourceType.DESCRIPTION) - _add_operation(random_lang, Operation.ADD, key, _predicate=ResourceType.LANGUAGE) - - -@db_session -def generate_torrent(metadata_store, db, parent, title=None): - infohash = random_infohash() - - # Give each torrent some health information. For now, we assume all torrents are healthy. - now = int(time.time()) - last_check = now - random.randint(3600, 24 * 3600) - category = random.choice(["Video", "Audio", "Documents", "Compressed", "Books", "Science"]) - torrent_state = metadata_store.TorrentState(infohash=infohash, seeders=10, last_check=last_check) - metadata_store.TorrentMetadata(title=title or generate_title(words_count=4), infohash=infohash, - origin_id=parent.id_, health=torrent_state, tags=category) - - tag_torrent(infohash, db) diff --git a/src/tribler/core/components/restapi/rest/statistics_endpoint.py b/src/tribler/core/components/restapi/rest/statistics_endpoint.py index 8e2a197407c..888ffad88c8 100644 --- a/src/tribler/core/components/restapi/rest/statistics_endpoint.py +++ b/src/tribler/core/components/restapi/rest/statistics_endpoint.py @@ -4,7 +4,7 @@ from ipv8.types import IPv8 from marshmallow.fields import Integer, String -from tribler.core.components.metadata_store.db.store import MetadataStore +from tribler.core.components.database.db.store import MetadataStore from tribler.core.components.restapi.rest.rest_endpoint import RESTEndpoint, RESTResponse from tribler.core.utilities.utilities import froze_it diff --git a/src/tribler/core/components/restapi/restapi_component.py b/src/tribler/core/components/restapi/restapi_component.py index 1dd90baaca8..a43ff2bdf1e 100644 --- a/src/tribler/core/components/restapi/restapi_component.py +++ b/src/tribler/core/components/restapi/restapi_component.py @@ -2,12 +2,13 @@ from typing import Type from ipv8.REST.root_endpoint import RootEndpoint as IPV8RootEndpoint - from tribler.core.components.bandwidth_accounting.bandwidth_accounting_component import BandwidthAccountingComponent from tribler.core.components.bandwidth_accounting.restapi.bandwidth_endpoint import BandwidthEndpoint from tribler.core.components.component import Component from tribler.core.components.content_discovery.content_discovery_component import ContentDiscoveryComponent +from tribler.core.components.content_discovery.restapi.search_endpoint import SearchEndpoint from tribler.core.components.database.database_component import DatabaseComponent +from tribler.core.components.database.restapi.database_endpoint import DatabaseEndpoint from tribler.core.components.exceptions import NoneComponent from tribler.core.components.ipv8.ipv8_component import Ipv8Component from tribler.core.components.key.key_component import KeyComponent @@ -18,9 +19,6 @@ from tribler.core.components.libtorrent.restapi.downloads_endpoint import DownloadsEndpoint from tribler.core.components.libtorrent.restapi.libtorrent_endpoint import LibTorrentEndpoint from tribler.core.components.libtorrent.restapi.torrentinfo_endpoint import TorrentInfoEndpoint -from tribler.core.components.metadata_store.metadata_store_component import MetadataStoreComponent -from tribler.core.components.metadata_store.restapi.metadata_endpoint import MetadataEndpoint -from tribler.core.components.metadata_store.restapi.search_endpoint import SearchEndpoint from tribler.core.components.reporter.exception_handler import CoreExceptionHandler, default_core_exception_handler from tribler.core.components.reporter.reported_error import ReportedError from tribler.core.components.reporter.reporter_component import ReporterComponent @@ -69,7 +67,6 @@ async def run(self): shutdown_event = session.shutdown_event log_dir = config.general.get_path_as_absolute('log_dir', config.state_dir) - metadata_store_component = await self.maybe_component(MetadataStoreComponent) key_component = await self.maybe_component(KeyComponent) ipv8_component = await self.maybe_component(Ipv8Component) @@ -98,16 +95,15 @@ async def run(self): core_exception_handler=self._core_exception_handler) self.maybe_add(BandwidthEndpoint, bandwidth_accounting_component.community) self.maybe_add(DownloadsEndpoint, libtorrent_component.download_manager, - metadata_store=metadata_store_component.mds, tunnel_community=tunnel_community) + metadata_store=db_component.mds, tunnel_community=tunnel_community) self.maybe_add(CreateTorrentEndpoint, libtorrent_component.download_manager) - self.maybe_add(StatisticsEndpoint, ipv8=ipv8_component.ipv8, metadata_store=metadata_store_component.mds) + self.maybe_add(StatisticsEndpoint, ipv8=ipv8_component.ipv8, metadata_store=db_component.mds) self.maybe_add(LibTorrentEndpoint, libtorrent_component.download_manager) self.maybe_add(TorrentInfoEndpoint, libtorrent_component.download_manager) - self.maybe_add(MetadataEndpoint, libtorrent_component.download_manager, torrent_checker, - metadata_store_component.mds, tribler_db=db_component.db, + self.maybe_add(DatabaseEndpoint, libtorrent_component.download_manager, torrent_checker, + db_component.mds, tribler_db=db_component.db, tag_rules_processor=knowledge_component.rules_processor) - self.maybe_add(SearchEndpoint, content_discovery_component.community, - metadata_store_component.mds, tribler_db=db_component.db) + self.maybe_add(SearchEndpoint, content_discovery_component.community) self.maybe_add(KnowledgeEndpoint, db=db_component.db, community=knowledge_component.community) if not isinstance(ipv8_component, NoneComponent): diff --git a/src/tribler/core/components/restapi/tests/test_restapi_component.py b/src/tribler/core/components/restapi/tests/test_restapi_component.py index 501a600bbc4..21991bdcc94 100644 --- a/src/tribler/core/components/restapi/tests/test_restapi_component.py +++ b/src/tribler/core/components/restapi/tests/test_restapi_component.py @@ -9,7 +9,6 @@ from tribler.core.components.key.key_component import KeyComponent from tribler.core.components.knowledge.knowledge_component import KnowledgeComponent from tribler.core.components.libtorrent.libtorrent_component import LibtorrentComponent -from tribler.core.components.metadata_store.metadata_store_component import MetadataStoreComponent from tribler.core.components.reporter.reported_error import ReportedError from tribler.core.components.resource_monitor.resource_monitor_component import ResourceMonitorComponent from tribler.core.components.restapi.rest.rest_endpoint import RESTEndpoint @@ -21,8 +20,7 @@ # pylint: disable=protected-access, not-callable, redefined-outer-name async def test_rest_component(tribler_config): components = [KeyComponent(), RESTComponent(), Ipv8Component(), LibtorrentComponent(), ResourceMonitorComponent(), - BandwidthAccountingComponent(), KnowledgeComponent(), SocksServersComponent(), - MetadataStoreComponent(), DatabaseComponent()] + BandwidthAccountingComponent(), KnowledgeComponent(), SocksServersComponent(), DatabaseComponent()] async with Session(tribler_config, components) as session: # Test REST component starts normally comp = session.get_instance(RESTComponent) diff --git a/src/tribler/core/components/tests/__init__.py b/src/tribler/core/components/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/tribler/core/components/torrent_checker/tests/test_torrent_checker_component.py b/src/tribler/core/components/torrent_checker/tests/test_torrent_checker_component.py index b66880a811a..49c7349500b 100644 --- a/src/tribler/core/components/torrent_checker/tests/test_torrent_checker_component.py +++ b/src/tribler/core/components/torrent_checker/tests/test_torrent_checker_component.py @@ -3,7 +3,6 @@ from tribler.core.components.key.key_component import KeyComponent from tribler.core.components.knowledge.knowledge_component import KnowledgeComponent from tribler.core.components.libtorrent.libtorrent_component import LibtorrentComponent -from tribler.core.components.metadata_store.metadata_store_component import MetadataStoreComponent from tribler.core.components.session import Session from tribler.core.components.socks_servers.socks_servers_component import SocksServersComponent from tribler.core.components.torrent_checker.torrent_checker_component import TorrentCheckerComponent @@ -12,7 +11,7 @@ # pylint: disable=protected-access async def test_torrent_checker_component(tribler_config): components = [DatabaseComponent(), SocksServersComponent(), LibtorrentComponent(), KeyComponent(), - Ipv8Component(), KnowledgeComponent(), MetadataStoreComponent(), TorrentCheckerComponent()] + Ipv8Component(), KnowledgeComponent(), TorrentCheckerComponent()] async with Session(tribler_config, components) as session: comp = session.get_instance(TorrentCheckerComponent) assert comp.started_event.is_set() and not comp.failed diff --git a/src/tribler/core/components/torrent_checker/torrent_checker/torrent_checker.py b/src/tribler/core/components/torrent_checker/torrent_checker/torrent_checker.py index c124e94e054..4dc705cb184 100644 --- a/src/tribler/core/components/torrent_checker/torrent_checker/torrent_checker.py +++ b/src/tribler/core/components/torrent_checker/torrent_checker/torrent_checker.py @@ -12,8 +12,8 @@ from tribler.core import notifications from tribler.core.components.libtorrent.download_manager.download_manager import DownloadManager -from tribler.core.components.metadata_store.db.serialization import REGULAR_TORRENT -from tribler.core.components.metadata_store.db.store import MetadataStore +from tribler.core.components.database.db.serialization import REGULAR_TORRENT +from tribler.core.components.database.db.store import MetadataStore from tribler.core.components.torrent_checker.torrent_checker import DHT from tribler.core.components.torrent_checker.torrent_checker.dataclasses import HEALTH_FRESHNESS_SECONDS, HealthInfo, \ TrackerResponse diff --git a/src/tribler/core/components/torrent_checker/torrent_checker/tracker_manager.py b/src/tribler/core/components/torrent_checker/torrent_checker/tracker_manager.py index 8bfaec0a085..d93ac3f5e94 100644 --- a/src/tribler/core/components/torrent_checker/torrent_checker/tracker_manager.py +++ b/src/tribler/core/components/torrent_checker/torrent_checker/tracker_manager.py @@ -4,7 +4,7 @@ from pony.orm import count, db_session -from tribler.core.components.metadata_store.db.store import MetadataStore +from tribler.core.components.database.db.store import MetadataStore from tribler.core.utilities.tracker_utils import get_uniformed_tracker_url MAX_TRACKER_FAILURES = 5 # if a tracker fails this amount of times in a row, its 'is_alive' will be marked as 0 (dead). diff --git a/src/tribler/core/components/torrent_checker/torrent_checker_component.py b/src/tribler/core/components/torrent_checker/torrent_checker_component.py index 1733aa5989d..14903dd6cb0 100644 --- a/src/tribler/core/components/torrent_checker/torrent_checker_component.py +++ b/src/tribler/core/components/torrent_checker/torrent_checker_component.py @@ -1,6 +1,6 @@ from tribler.core.components.component import Component +from tribler.core.components.database.database_component import DatabaseComponent from tribler.core.components.libtorrent.libtorrent_component import LibtorrentComponent -from tribler.core.components.metadata_store.metadata_store_component import MetadataStoreComponent from tribler.core.components.socks_servers.socks_servers_component import SocksServersComponent from tribler.core.components.torrent_checker.torrent_checker.torrent_checker import TorrentChecker from tribler.core.components.torrent_checker.torrent_checker.tracker_manager import TrackerManager @@ -14,17 +14,17 @@ async def run(self): config = self.session.config - metadata_store_component = await self.require_component(MetadataStoreComponent) + database_component = await self.require_component(DatabaseComponent) libtorrent_component = await self.require_component(LibtorrentComponent) socks_servers_component = await self.require_component(SocksServersComponent) - tracker_manager = TrackerManager(state_dir=config.state_dir, metadata_store=metadata_store_component.mds) + tracker_manager = TrackerManager(state_dir=config.state_dir, metadata_store=database_component.mds) torrent_checker = TorrentChecker(config=config, download_manager=libtorrent_component.download_manager, notifier=self.session.notifier, tracker_manager=tracker_manager, socks_listen_ports=socks_servers_component.socks_ports, - metadata_store=metadata_store_component.mds) + metadata_store=database_component.mds) self.torrent_checker = torrent_checker await torrent_checker.initialize() diff --git a/src/tribler/core/config/tribler_config.py b/src/tribler/core/config/tribler_config.py index cd3dc0ae6e4..ccee0b92a30 100644 --- a/src/tribler/core/config/tribler_config.py +++ b/src/tribler/core/config/tribler_config.py @@ -19,7 +19,7 @@ from tribler.core.components.key.settings import TrustchainSettings from tribler.core.components.libtorrent.settings import DownloadDefaultsSettings, LibtorrentSettings from tribler.core.components.content_discovery.settings import ContentDiscoveryComponentConfig -from tribler.core.components.metadata_store.settings import ChantSettings +from tribler.core.components.database.settings import ChantSettings from tribler.core.components.resource_monitor.settings import ResourceMonitorSettings from tribler.core.components.restapi.rest.settings import APISettings from tribler.core.components.torrent_checker.settings import TorrentCheckerSettings diff --git a/src/tribler/core/start_core.py b/src/tribler/core/start_core.py index dd4120dd02b..23502f43d0f 100644 --- a/src/tribler/core/start_core.py +++ b/src/tribler/core/start_core.py @@ -21,7 +21,6 @@ from tribler.core.components.key.key_component import KeyComponent from tribler.core.components.knowledge.knowledge_component import KnowledgeComponent from tribler.core.components.libtorrent.libtorrent_component import LibtorrentComponent -from tribler.core.components.metadata_store.metadata_store_component import MetadataStoreComponent from tribler.core.components.payout.payout_component import PayoutComponent from tribler.core.components.content_discovery.content_discovery_component import ContentDiscoveryComponent from tribler.core.components.reporter.exception_handler import default_core_exception_handler @@ -58,8 +57,6 @@ def components_gen(config: TriblerConfig): yield GuiProcessWatcherComponent() yield DatabaseComponent() yield RESTComponent() - if config.chant.enabled or config.torrent_checking.enabled: - yield MetadataStoreComponent() if config.ipv8.enabled: yield Ipv8Component() diff --git a/src/tribler/core/tests/__init__.py b/src/tribler/core/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/tribler/core/upgrade/db8_to_db10.py b/src/tribler/core/upgrade/db8_to_db10.py index 68534b56456..4fb688d1a2f 100644 --- a/src/tribler/core/upgrade/db8_to_db10.py +++ b/src/tribler/core/upgrade/db8_to_db10.py @@ -6,7 +6,7 @@ from pony.orm import db_session -from tribler.core.components.metadata_store.db.store import MetadataStore +from tribler.core.components.database.db.store import MetadataStore from tribler.core.utilities.db_corruption_handling import sqlite_replacement TABLE_NAMES = ( diff --git a/src/tribler/core/upgrade/tests/test_upgrader.py b/src/tribler/core/upgrade/tests/test_upgrader.py index d04610be0c6..eea1831b546 100644 --- a/src/tribler/core/upgrade/tests/test_upgrader.py +++ b/src/tribler/core/upgrade/tests/test_upgrader.py @@ -10,8 +10,8 @@ from pony.orm import db_session, select from tribler.core.components.bandwidth_accounting.db.database import BandwidthDatabase -from tribler.core.components.metadata_store.db.orm_bindings.torrent_metadata import CHANNEL_DIR_NAME_LENGTH -from tribler.core.components.metadata_store.db.store import CURRENT_DB_VERSION, MetadataStore +from tribler.core.components.database.db.orm_bindings.torrent_metadata import CHANNEL_DIR_NAME_LENGTH +from tribler.core.components.database.db.store import CURRENT_DB_VERSION, MetadataStore from tribler.core.tests.tools.common import TESTS_DATA_DIR from tribler.core.upgrade.db8_to_db10 import calc_progress from tribler.core.upgrade.tags_to_knowledge.previous_dbs.tags_db import TagDatabase diff --git a/src/tribler/core/upgrade/upgrade.py b/src/tribler/core/upgrade/upgrade.py index 884393a877f..3d7be8a921f 100644 --- a/src/tribler/core/upgrade/upgrade.py +++ b/src/tribler/core/upgrade/upgrade.py @@ -11,8 +11,8 @@ from pony.orm import db_session, delete from tribler.core.components.bandwidth_accounting.db.database import BandwidthDatabase -from tribler.core.components.metadata_store.db.orm_bindings.torrent_metadata import CHANNEL_DIR_NAME_LENGTH -from tribler.core.components.metadata_store.db.store import ( +from tribler.core.components.database.db.orm_bindings.torrent_metadata import CHANNEL_DIR_NAME_LENGTH +from tribler.core.components.database.db.store import ( CURRENT_DB_VERSION, MetadataStore, sql_create_partial_index_torrentstate_last_check, ) diff --git a/src/tribler/gui/tests/test_gui.py b/src/tribler/gui/tests/test_gui.py index 4ec71cfefd3..350c25bc03a 100644 --- a/src/tribler/gui/tests/test_gui.py +++ b/src/tribler/gui/tests/test_gui.py @@ -15,7 +15,7 @@ from tribler.core.components.reporter.reported_error import ReportedError from tribler.core.sentry_reporter.sentry_reporter import SentryReporter from tribler.core.tests.tools.common import TESTS_DATA_DIR -from tribler.core.utilities.process_manager import ProcessKind, ProcessManager, TriblerProcess +from tribler.core.utilities.process_manager import ProcessKind, ProcessManager from tribler.core.utilities.rest_utils import path_to_url from tribler.core.utilities.unicode import hexlify from tribler.gui.app_manager import AppManager diff --git a/src/tribler/gui/tribler_window.py b/src/tribler/gui/tribler_window.py index 0d67e65d7d3..c33297b309e 100644 --- a/src/tribler/gui/tribler_window.py +++ b/src/tribler/gui/tribler_window.py @@ -611,7 +611,7 @@ def on_search_text_change(self, text): # We do not want to bother the database on petty 1-character queries if len(text) < 2: return - request_manager.get("search/completions", self.on_received_search_completions, url_params={'q': text}) + request_manager.get("metadata/search/completions", self.on_received_search_completions, url_params={'q': text}) def on_received_search_completions(self, completions): if completions is None: diff --git a/src/tribler/gui/widgets/channelcontentswidget.py b/src/tribler/gui/widgets/channelcontentswidget.py index 4831d3f4b2d..52c2507f1bf 100644 --- a/src/tribler/gui/widgets/channelcontentswidget.py +++ b/src/tribler/gui/widgets/channelcontentswidget.py @@ -6,8 +6,8 @@ from PyQt5.QtWidgets import QAction, QFileDialog from psutil import LINUX -from tribler.core.components.metadata_store.db.orm_bindings.torrent_metadata import NEW -from tribler.core.components.metadata_store.db.serialization import CHANNEL_TORRENT, COLLECTION_NODE +from tribler.core.components.database.db.orm_bindings.torrent_metadata import NEW +from tribler.core.components.database.db.serialization import CHANNEL_TORRENT, COLLECTION_NODE from tribler.core.utilities.simpledefs import CHANNEL_STATE from tribler.gui.defs import ( BUTTON_TYPE_CONFIRM, diff --git a/src/tribler/gui/widgets/lazytableview.py b/src/tribler/gui/widgets/lazytableview.py index fa966149a5e..7de55c476cf 100644 --- a/src/tribler/gui/widgets/lazytableview.py +++ b/src/tribler/gui/widgets/lazytableview.py @@ -5,7 +5,7 @@ from PyQt5.QtGui import QGuiApplication, QMouseEvent, QMovie from PyQt5.QtWidgets import QAbstractItemView, QApplication, QHeaderView, QLabel, QTableView -from tribler.core.components.metadata_store.db.serialization import SNIPPET +from tribler.core.components.database.db.serialization import SNIPPET from tribler.gui.dialogs.editmetadatadialog import EditMetadataDialog from tribler.gui.network.request_manager import request_manager from tribler.gui.utilities import connect, data_item2uri, get_image_path, index2uri diff --git a/src/tribler/gui/widgets/searchresultswidget.py b/src/tribler/gui/widgets/searchresultswidget.py index e7112ff4b44..d3941d8489c 100644 --- a/src/tribler/gui/widgets/searchresultswidget.py +++ b/src/tribler/gui/widgets/searchresultswidget.py @@ -5,7 +5,7 @@ from PyQt5 import uic -from tribler.core.components.metadata_store.db.serialization import REGULAR_TORRENT +from tribler.core.components.database.db.serialization import REGULAR_TORRENT from tribler.core.utilities.utilities import Query, to_fts_query from tribler.gui.network.request_manager import request_manager from tribler.gui.sentry_mixin import AddBreadcrumbOnShowMixin @@ -92,7 +92,7 @@ def search(self, query: Query) -> bool: self.last_search_time = time.time() model = SearchResultsModel( - endpoint_url="search/local", + endpoint_url="metadata/search/local", hide_xxx=self.results_page_content.hide_xxx, original_query=query.original_query, text_filter=to_fts_query(query.fts_text), diff --git a/src/tribler/gui/widgets/tablecontentdelegate.py b/src/tribler/gui/widgets/tablecontentdelegate.py index 005171674aa..7c48cb3f4d0 100644 --- a/src/tribler/gui/widgets/tablecontentdelegate.py +++ b/src/tribler/gui/widgets/tablecontentdelegate.py @@ -7,16 +7,14 @@ from psutil import LINUX from tribler.core.components.database.db.layers.knowledge_data_access_layer import ResourceType -from tribler.core.components.metadata_store.db.serialization import CHANNEL_TORRENT, COLLECTION_NODE, REGULAR_TORRENT, \ +from tribler.core.components.database.db.serialization import COLLECTION_NODE, REGULAR_TORRENT, \ SNIPPET -from tribler.core.utilities.simpledefs import CHANNEL_STATE from tribler.gui.defs import ( COMMIT_STATUS_COMMITTED, COMMIT_STATUS_NEW, COMMIT_STATUS_TODELETE, COMMIT_STATUS_UPDATED, ContentCategories, - DARWIN, HEALTH_CHECKING, HEALTH_DEAD, HEALTH_ERROR, @@ -30,9 +28,8 @@ TAG_TEXT_COLOR, TAG_TEXT_HORIZONTAL_PADDING, TAG_TOP_MARGIN, - WINDOWS, ) -from tribler.gui.utilities import format_votes, get_color, get_gui_setting, get_health, get_image_path, \ +from tribler.gui.utilities import get_color, get_gui_setting, get_health, get_image_path, \ get_objects_with_predicate, tr from tribler.gui.widgets.tablecontentmodel import Column, RemoteTableModel from tribler.gui.widgets.tableiconbuttons import DownloadIconButton diff --git a/src/tribler/gui/widgets/tablecontentmodel.py b/src/tribler/gui/widgets/tablecontentmodel.py index cd057d7f7a8..6053fdaff82 100644 --- a/src/tribler/gui/widgets/tablecontentmodel.py +++ b/src/tribler/gui/widgets/tablecontentmodel.py @@ -10,7 +10,7 @@ from PyQt5.QtCore import QAbstractTableModel, QModelIndex, QRectF, QSize, QTimerEvent, Qt, pyqtSignal -from tribler.core.components.metadata_store.db.serialization import COLLECTION_NODE, REGULAR_TORRENT, SNIPPET +from tribler.core.components.database.db.serialization import COLLECTION_NODE, REGULAR_TORRENT, SNIPPET from tribler.core.utilities.search_utils import item_rank from tribler.core.utilities.simpledefs import CHANNEL_STATE from tribler.core.utilities.utilities import to_fts_query diff --git a/src/tribler/gui/widgets/triblertablecontrollers.py b/src/tribler/gui/widgets/triblertablecontrollers.py index e130036835e..aa0d8fe5911 100644 --- a/src/tribler/gui/widgets/triblertablecontrollers.py +++ b/src/tribler/gui/widgets/triblertablecontrollers.py @@ -2,7 +2,6 @@ This file contains various controllers for table views. The responsibility of the controller is to populate the table view with some data, contained in a specific model. """ -import json import logging from PyQt5.QtCore import QObject, QTimer, Qt @@ -10,7 +9,7 @@ from PyQt5.QtNetwork import QNetworkRequest from PyQt5.QtWidgets import QAction -from tribler.core.components.metadata_store.db.serialization import REGULAR_TORRENT +from tribler.core.components.database.db.serialization import REGULAR_TORRENT from tribler.gui.defs import HEALTH_CHECKING, HEALTH_UNCHECKED from tribler.gui.network.request_manager import request_manager from tribler.gui.tribler_action_menu import TriblerActionMenu