From a643e2b9e39cdf334470144135374c2e1a631c91 Mon Sep 17 00:00:00 2001 From: drew2a Date: Tue, 30 May 2023 16:17:05 +0200 Subject: [PATCH 1/2] Decrease `MIN_RESOURCE_LENGTH` down to 2 to be able to gossip languages --- .../knowledge/community/tests/test_knowledge_validator.py | 3 ++- src/tribler/core/components/knowledge/knowledge_constants.py | 2 +- .../components/knowledge/rules/tests/test_tag_rules_base.py | 2 +- src/tribler/gui/tests/test_gui.py | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/tribler/core/components/knowledge/community/tests/test_knowledge_validator.py b/src/tribler/core/components/knowledge/community/tests/test_knowledge_validator.py index 9256c1c788d..a4159822b05 100644 --- a/src/tribler/core/components/knowledge/community/tests/test_knowledge_validator.py +++ b/src/tribler/core/components/knowledge/community/tests/test_knowledge_validator.py @@ -5,6 +5,7 @@ from tribler.core.components.knowledge.db.knowledge_db import Operation, ResourceType VALID_TAGS = [ + 'nl', 'tag', 'Tag', 'Тэг', @@ -13,7 +14,7 @@ INVALID_TAGS = [ '', - 'ta', # less than 3 + 't', 't' * 51, # more than 50 ] diff --git a/src/tribler/core/components/knowledge/knowledge_constants.py b/src/tribler/core/components/knowledge/knowledge_constants.py index 88d6778bb59..352fd0a8e26 100644 --- a/src/tribler/core/components/knowledge/knowledge_constants.py +++ b/src/tribler/core/components/knowledge/knowledge_constants.py @@ -1,2 +1,2 @@ -MIN_RESOURCE_LENGTH = 3 +MIN_RESOURCE_LENGTH = 2 MAX_RESOURCE_LENGTH = 50 diff --git a/src/tribler/core/components/knowledge/rules/tests/test_tag_rules_base.py b/src/tribler/core/components/knowledge/rules/tests/test_tag_rules_base.py index ac1560c69c7..6462fb0f2d5 100644 --- a/src/tribler/core/components/knowledge/rules/tests/test_tag_rules_base.py +++ b/src/tribler/core/components/knowledge/rules/tests/test_tag_rules_base.py @@ -4,4 +4,4 @@ def test_extract_only_valid_tags(): # test that extract_only_valid_tags extracts only valid tags - assert set(extract_only_valid_tags('[valid-tag, in va li d]', rules=general_rules)) == {'valid-tag'} + assert set(extract_only_valid_tags('[valid-tag, i n v a l i d]', rules=general_rules)) == {'valid-tag'} diff --git a/src/tribler/gui/tests/test_gui.py b/src/tribler/gui/tests/test_gui.py index 0698a7d5272..c942d85d64d 100644 --- a/src/tribler/gui/tests/test_gui.py +++ b/src/tribler/gui/tests/test_gui.py @@ -696,7 +696,7 @@ def test_tags_dialog(window): assert widget.content_table.add_tags_dialog.dialog_widget.error_text_label.isVisible() QTest.keyClick(tags_input, "c") - assert tags_input.tags[-1].text == "aac" + assert tags_input.tags[-1].text == "ac" # Test creating a new tag by clicking to the right of the right-most tag QTest.mouseClick(tags_input, Qt.LeftButton, pos=tags_input.tags[-1].rect.topRight().toPoint() + QPoint(10, 0)) From 706d6ca2695b19b053526af90e241dd0d3a24187 Mon Sep 17 00:00:00 2001 From: drew2a Date: Tue, 30 May 2023 23:00:20 +0200 Subject: [PATCH 2/2] Add `CONTENT_ITEM` relation --- .../components/knowledge/db/knowledge_db.py | 1 + .../knowledge/db/tests/test_knowledge_db.py | 24 ++++++++++--------- .../knowledge/restapi/knowledge_endpoint.py | 18 +++++++------- .../rules/knowledge_rules_processor.py | 5 ++-- .../metadata_store/restapi/search_endpoint.py | 2 +- src/tribler/gui/dialogs/editmetadatadialog.py | 7 +++++- .../gui/qt_resources/edit_metadata_dialog.ui | 2 +- 7 files changed, 34 insertions(+), 25 deletions(-) diff --git a/src/tribler/core/components/knowledge/db/knowledge_db.py b/src/tribler/core/components/knowledge/db/knowledge_db.py index b7adbeced68..b6497147af6 100644 --- a/src/tribler/core/components/knowledge/db/knowledge_db.py +++ b/src/tribler/core/components/knowledge/db/knowledge_db.py @@ -51,6 +51,7 @@ class ResourceType(IntEnum): # this is a section for extra types TAG = 101 TORRENT = 102 + CONTENT_ITEM = 103 @dataclass diff --git a/src/tribler/core/components/knowledge/db/tests/test_knowledge_db.py b/src/tribler/core/components/knowledge/db/tests/test_knowledge_db.py index 471ce15d215..bc1b2fbd663 100644 --- a/src/tribler/core/components/knowledge/db/tests/test_knowledge_db.py +++ b/src/tribler/core/components/knowledge/db/tests/test_knowledge_db.py @@ -539,24 +539,26 @@ def test_get_subjects(self): self.db, { 'infohash1': [ - Resource(predicate=ResourceType.TITLE, name='ubuntu', auto_generated=True), + Resource(predicate=ResourceType.CONTENT_ITEM, name='ubuntu', auto_generated=True), Resource(predicate=ResourceType.TAG, name='linux', auto_generated=True), ], 'infohash2': [ - Resource(predicate=ResourceType.TITLE, name='ubuntu', auto_generated=True), + Resource(predicate=ResourceType.CONTENT_ITEM, name='ubuntu', auto_generated=True), Resource(predicate=ResourceType.TAG, name='linux', auto_generated=True), ], 'infohash3': [ - Resource(predicate=ResourceType.TITLE, name='debian', auto_generated=True), + Resource(predicate=ResourceType.CONTENT_ITEM, name='debian', auto_generated=True), Resource(predicate=ResourceType.TAG, name='linux', auto_generated=True), ], } ) - actual = self.db.get_subjects(subject_type=ResourceType.TORRENT, predicate=ResourceType.TITLE, obj='missed') + actual = self.db.get_subjects(subject_type=ResourceType.TORRENT, predicate=ResourceType.CONTENT_ITEM, + obj='missed') assert actual == [] - actual = self.db.get_subjects(subject_type=ResourceType.TORRENT, predicate=ResourceType.TITLE, obj='ubuntu') + actual = self.db.get_subjects(subject_type=ResourceType.TORRENT, predicate=ResourceType.CONTENT_ITEM, + obj='ubuntu') assert actual == ['infohash1', 'infohash2'] actual = self.db.get_subjects(subject_type=ResourceType.TORRENT, predicate=ResourceType.TAG, obj='linux') @@ -568,11 +570,11 @@ def test_get_statements(self): self.db, { 'infohash1': [ - Resource(predicate=ResourceType.TITLE, name='ubuntu', auto_generated=True), + Resource(predicate=ResourceType.CONTENT_ITEM, name='ubuntu', auto_generated=True), Resource(predicate=ResourceType.TYPE, name='linux', auto_generated=True), ], 'infohash2': [ - Resource(predicate=ResourceType.TITLE, name='debian', auto_generated=True), + Resource(predicate=ResourceType.CONTENT_ITEM, name='debian', auto_generated=True), Resource(predicate=ResourceType.TYPE, name='linux', auto_generated=True), ], 'INFOHASH1': [ @@ -582,7 +584,7 @@ def test_get_statements(self): ) expected = [ - SimpleStatement(subject_type=ResourceType.TORRENT, subject='infohash1', predicate=ResourceType.TITLE, + SimpleStatement(subject_type=ResourceType.TORRENT, subject='infohash1', predicate=ResourceType.CONTENT_ITEM, object='ubuntu'), SimpleStatement(subject_type=ResourceType.TORRENT, subject='infohash1', predicate=ResourceType.TYPE, object='linux') @@ -600,11 +602,11 @@ def test_various_queries(self): self.db, { 'infohash1': [ - Resource(predicate=ResourceType.TITLE, name='ubuntu'), + Resource(predicate=ResourceType.CONTENT_ITEM, name='ubuntu'), Resource(predicate=ResourceType.TYPE, name='linux'), ], 'infohash2': [ - Resource(predicate=ResourceType.TITLE, name='debian'), + Resource(predicate=ResourceType.CONTENT_ITEM, name='debian'), Resource(predicate=ResourceType.TYPE, name='linux'), ], 'infohash3': [ @@ -632,7 +634,7 @@ def _subjects(subject_type=None, obj='', predicate=None): assert _subjects(obj='linux') == {'infohash1', 'infohash2', 'infohash3'} assert _subjects(predicate=ResourceType.TAG, obj='linux') == {'infohash3'} - assert _subjects(predicate=ResourceType.TITLE) == {'infohash1', 'infohash2'} + assert _subjects(predicate=ResourceType.CONTENT_ITEM) == {'infohash1', 'infohash2'} @db_session def test_non_existent_misc(self): diff --git a/src/tribler/core/components/knowledge/restapi/knowledge_endpoint.py b/src/tribler/core/components/knowledge/restapi/knowledge_endpoint.py index 64f0a26981a..d43acb145b4 100644 --- a/src/tribler/core/components/knowledge/restapi/knowledge_endpoint.py +++ b/src/tribler/core/components/knowledge/restapi/knowledge_endpoint.py @@ -9,8 +9,8 @@ from tribler.core.components.knowledge.community.knowledge_community import KnowledgeCommunity from tribler.core.components.knowledge.community.knowledge_payload import StatementOperation +from tribler.core.components.knowledge.community.knowledge_validator import is_valid_resource from tribler.core.components.knowledge.db.knowledge_db import KnowledgeDatabase, Operation, ResourceType -from tribler.core.components.knowledge.knowledge_constants import MAX_RESOURCE_LENGTH, MIN_RESOURCE_LENGTH from tribler.core.components.restapi.rest.rest_endpoint import HTTP_BAD_REQUEST, RESTEndpoint, RESTResponse from tribler.core.components.restapi.rest.schema import HandledErrorSchema from tribler.core.utilities.utilities import froze_it @@ -65,24 +65,21 @@ async def update_knowledge_entries(self, request): return error_response # Validate whether the size of the tag is within the allowed range and filter out duplicate tags. - tags = set() statements = [] + self._logger.info(f'Statements about {infohash}: {params["statements"]}') for statement in params["statements"]: obj = statement["object"] - if statement["predicate"] == ResourceType.TAG and \ - (len(obj) < MIN_RESOURCE_LENGTH or len(obj) > MAX_RESOURCE_LENGTH): + if not is_valid_resource(obj): return RESTResponse({"error": "Invalid tag length"}, status=HTTP_BAD_REQUEST) - if obj not in tags: - tags.add(obj) - statements.append(statement) + statements.append(statement) self.modify_statements(infohash, statements) return RESTResponse({"success": True}) @db_session - def modify_statements(self, infohash: str, statements): + def modify_statements(self, infohash: str, statements: list): """ Modify the statements of a particular content item. """ @@ -92,7 +89,9 @@ def modify_statements(self, infohash: str, statements): # First, get the current statements and compute the diff between the old and new statements old_statements = self.db.get_statements(subject_type=ResourceType.TORRENT, subject=infohash) old_statements = {(stmt.predicate, stmt.object) for stmt in old_statements} + self._logger.info(f'Old statements: {old_statements}') new_statements = {(stmt["predicate"], stmt["object"]) for stmt in statements} + self._logger.info(f'New statements: {new_statements}') added_statements = new_statements - old_statements removed_statements = old_statements - new_statements @@ -109,6 +108,9 @@ def modify_statements(self, infohash: str, statements): signature = self.community.sign(operation) self.db.add_operation(operation, signature, is_local_peer=True) + self._logger.info(f'Added statements: {added_statements}') + self._logger.info(f'Removed statements: {removed_statements}') + @docs( tags=["General"], summary="Get tag suggestions for a torrent with a particular infohash.", diff --git a/src/tribler/core/components/knowledge/rules/knowledge_rules_processor.py b/src/tribler/core/components/knowledge/rules/knowledge_rules_processor.py index 6c8b55288cb..5f6f018fad1 100644 --- a/src/tribler/core/components/knowledge/rules/knowledge_rules_processor.py +++ b/src/tribler/core/components/knowledge/rules/knowledge_rules_processor.py @@ -1,7 +1,6 @@ import logging import queue import time -from asyncio import Event from dataclasses import dataclass from typing import Optional, Set @@ -187,8 +186,8 @@ async def process_torrent_title(self, infohash: Optional[bytes] = None, title: O objects=tags) if content_items := set(extract_only_valid_tags(title, rules=content_items_rules)): - self.save_statements(subject_type=ResourceType.TORRENT, subject=infohash_str, predicate=ResourceType.TITLE, - objects=content_items) + self.save_statements(subject_type=ResourceType.TORRENT, subject=infohash_str, + predicate=ResourceType.CONTENT_ITEM, objects=content_items) return len(tags) + len(content_items) diff --git a/src/tribler/core/components/metadata_store/restapi/search_endpoint.py b/src/tribler/core/components/metadata_store/restapi/search_endpoint.py index dfbaef20d45..a1bbf3dc670 100644 --- a/src/tribler/core/components/metadata_store/restapi/search_endpoint.py +++ b/src/tribler/core/components/metadata_store/restapi/search_endpoint.py @@ -50,7 +50,7 @@ def build_snippets(self, search_results: List[Dict]) -> List[Dict]: with db_session: content_items: List[str] = self.knowledge_db.get_objects(subject_type=ResourceType.TORRENT, subject=search_result["infohash"], - predicate=ResourceType.TITLE) + predicate=ResourceType.CONTENT_ITEM) if content_items: for content_id in content_items: content_to_torrents[content_id].append(search_result) diff --git a/src/tribler/gui/dialogs/editmetadatadialog.py b/src/tribler/gui/dialogs/editmetadatadialog.py index b809606c251..32ada509b54 100644 --- a/src/tribler/gui/dialogs/editmetadatadialog.py +++ b/src/tribler/gui/dialogs/editmetadatadialog.py @@ -14,7 +14,12 @@ from tribler.gui.utilities import connect, get_objects_with_predicate, get_ui_file_path, tr from tribler.gui.widgets.tagbutton import TagButton -METADATA_TABLE_PREDICATES = [ResourceType.TITLE, ResourceType.DESCRIPTION, ResourceType.DATE, ResourceType.LANGUAGE] +METADATA_TABLE_PREDICATES = [ + ResourceType.CONTENT_ITEM, + ResourceType.DESCRIPTION, + ResourceType.DATE, + ResourceType.LANGUAGE +] class EditMetadataDialog(DialogContainer): diff --git a/src/tribler/gui/qt_resources/edit_metadata_dialog.ui b/src/tribler/gui/qt_resources/edit_metadata_dialog.ui index ab593852982..9fc177b6689 100644 --- a/src/tribler/gui/qt_resources/edit_metadata_dialog.ui +++ b/src/tribler/gui/qt_resources/edit_metadata_dialog.ui @@ -276,7 +276,7 @@ margin: 0px; - Title + Content Item