From c99e53eb350c28f2c1870fbe375391b55c1fee63 Mon Sep 17 00:00:00 2001 From: Sandip Pandey Date: Thu, 28 Jun 2018 17:21:18 +0200 Subject: [PATCH] Renamed popular community to popularity community and added identifier to subscription request --- .../Core/APIImplementation/LaunchManyCore.py | 4 +- Tribler/Core/Config/config.spec | 2 +- Tribler/Core/Config/tribler_config.py | 8 +- Tribler/Core/Modules/search_manager.py | 6 +- .../Core/TorrentChecker/torrent_checker.py | 8 +- .../{popular => popularity}/__init__.py | 0 .../{popular => popularity}/test_community.py | 80 ++++++------------- .../{popular => popularity}/test_payload.py | 10 ++- .../test_repository.py | 22 +++-- .../Test/Core/Config/test_tribler_config.py | 8 +- .../TorrentChecker/test_torrentchecker.py | 32 ++++---- Tribler/Test/test_as_server.py | 2 +- .../{popular => popularity}/__init__.py | 0 .../{popular => popularity}/community.py | 20 ++--- .../{popular => popularity}/constants.py | 0 .../{popular => popularity}/payload.py | 14 ++-- .../{popular => popularity}/pubsub.py | 29 ++++--- .../{popular => popularity}/repository.py | 2 +- .../{popular => popularity}/request.py | 5 +- 19 files changed, 114 insertions(+), 138 deletions(-) rename Tribler/Test/Community/{popular => popularity}/__init__.py (100%) rename Tribler/Test/Community/{popular => popularity}/test_community.py (90%) rename Tribler/Test/Community/{popular => popularity}/test_payload.py (95%) rename Tribler/Test/Community/{popular => popularity}/test_repository.py (95%) rename Tribler/community/{popular => popularity}/__init__.py (100%) rename Tribler/community/{popular => popularity}/community.py (93%) rename Tribler/community/{popular => popularity}/constants.py (100%) rename Tribler/community/{popular => popularity}/payload.py (96%) rename Tribler/community/{popular => popularity}/pubsub.py (92%) rename Tribler/community/{popular => popularity}/repository.py (98%) rename Tribler/community/{popular => popularity}/request.py (81%) diff --git a/Tribler/Core/APIImplementation/LaunchManyCore.py b/Tribler/Core/APIImplementation/LaunchManyCore.py index 6f7aa62dfb6..84078c98afb 100644 --- a/Tribler/Core/APIImplementation/LaunchManyCore.py +++ b/Tribler/Core/APIImplementation/LaunchManyCore.py @@ -289,8 +289,8 @@ def load_ipv8_overlays(self): self.ipv8.strategies.append((RandomWalk(self.market_community), 20)) # Popular Community - if self.session.config.get_popular_community_enabled(): - from Tribler.community.popular.community import PopularityCommunity + if self.session.config.get_popularity_community_enabled(): + from Tribler.community.popularity.community import PopularityCommunity local_peer = Peer(self.session.trustchain_keypair) diff --git a/Tribler/Core/Config/config.spec b/Tribler/Core/Config/config.spec index c514a0b0cf4..2205b6a0284 100644 --- a/Tribler/Core/Config/config.spec +++ b/Tribler/Core/Config/config.spec @@ -116,6 +116,6 @@ enabled = boolean(default=True) sources = string_list(default=list()) max_disk_space = integer(min=0, default=53687091200) -[popular_community] +[popularity_community] enabled = boolean(default=True) cache_dir = string(default=health_cache) \ No newline at end of file diff --git a/Tribler/Core/Config/tribler_config.py b/Tribler/Core/Config/tribler_config.py index 48133d14702..a779a770aae 100644 --- a/Tribler/Core/Config/tribler_config.py +++ b/Tribler/Core/Config/tribler_config.py @@ -531,11 +531,11 @@ def get_dummy_wallets_enabled(self): # Popular Community - def get_popular_community_enabled(self): - return self.config['popular_community']['enabled'] + def get_popularity_community_enabled(self): + return self.config['popularity_community']['enabled'] - def set_popular_community_enabled(self, value): - self.config['popular_community']['enabled'] = value + def set_popularity_community_enabled(self, value): + self.config['popularity_community']['enabled'] = value # Torrent store diff --git a/Tribler/Core/Modules/search_manager.py b/Tribler/Core/Modules/search_manager.py index 9c071bd255d..fcd4ebb697c 100644 --- a/Tribler/Core/Modules/search_manager.py +++ b/Tribler/Core/Modules/search_manager.py @@ -56,9 +56,9 @@ def search_for_torrents(self, keywords): break self._current_keywords = keywords - # If popular community is enabled, send the search request there as well - if self.session.lm.popular_community: - self.session.lm.popular_community.send_torrent_search_request(keywords) + # If popularity community is enabled, send the search request there as well + if self.session.lm.popularity_community: + self.session.lm.popularity_community.send_torrent_search_request(keywords) return nr_requests_made diff --git a/Tribler/Core/TorrentChecker/torrent_checker.py b/Tribler/Core/TorrentChecker/torrent_checker.py index fd01fcf14d2..890bb88ada6 100644 --- a/Tribler/Core/TorrentChecker/torrent_checker.py +++ b/Tribler/Core/TorrentChecker/torrent_checker.py @@ -13,7 +13,7 @@ from Tribler.Core.TorrentChecker.session import create_tracker_session, FakeDHTSession, UdpSocketManager from Tribler.Core.Utilities.tracker_utils import MalformedTrackerURLException from Tribler.Core.simpledefs import NTFY_TORRENTS -from Tribler.community.popular.repository import TYPE_TORRENT_HEALTH +from Tribler.community.popularity.repository import TYPE_TORRENT_HEALTH from Tribler.dispersy.util import blocking_call_on_reactor_thread, call_on_reactor_thread from Tribler.pyipv8.ipv8.taskmanager import TaskManager @@ -199,7 +199,7 @@ def on_gui_request_completed(self, infohash, result): self._update_torrent_result(torrent_update_dict) - # Add this result to popular community to publish to subscribers + # Add this result to popularity community to publish to subscribers self.publish_torrent_result(torrent_update_dict) return final_response @@ -336,7 +336,7 @@ def publish_torrent_result(self, response): self._logger.info("Not publishing zero seeded torrents") return content = (response['infohash'], response['seeders'], response['leechers'], response['last_check']) - if self.tribler_session.lm.popular_community: - self.tribler_session.lm.popular_community.queue_content(TYPE_TORRENT_HEALTH, content) + if self.tribler_session.lm.popularity_community: + self.tribler_session.lm.popularity_community.queue_content(TYPE_TORRENT_HEALTH, content) else: self._logger.info("Popular community not available to publish torrent checker result") diff --git a/Tribler/Test/Community/popular/__init__.py b/Tribler/Test/Community/popularity/__init__.py similarity index 100% rename from Tribler/Test/Community/popular/__init__.py rename to Tribler/Test/Community/popularity/__init__.py diff --git a/Tribler/Test/Community/popular/test_community.py b/Tribler/Test/Community/popularity/test_community.py similarity index 90% rename from Tribler/Test/Community/popular/test_community.py rename to Tribler/Test/Community/popularity/test_community.py index 6e57085ab20..3d695353e57 100644 --- a/Tribler/Test/Community/popular/test_community.py +++ b/Tribler/Test/Community/popularity/test_community.py @@ -2,24 +2,24 @@ import string from Tribler.Test.Core.base_test import MockObject -from Tribler.community.popular import constants -from Tribler.community.popular.community import PopularityCommunity, MSG_TORRENT_HEALTH_RESPONSE, \ +from Tribler.community.popularity import constants +from Tribler.community.popularity.community import PopularityCommunity, MSG_TORRENT_HEALTH_RESPONSE, \ MSG_CHANNEL_HEALTH_RESPONSE, ERROR_UNKNOWN_PEER, ERROR_NO_CONTENT, \ ERROR_UNKNOWN_RESPONSE -from Tribler.community.popular.constants import SEARCH_TORRENT_REQUEST, MSG_TORRENT_INFO_RESPONSE, MSG_SUBSCRIPTION -from Tribler.community.popular.payload import SearchResponseItemPayload, TorrentInfoResponsePayload, \ +from Tribler.community.popularity.constants import SEARCH_TORRENT_REQUEST, MSG_TORRENT_INFO_RESPONSE, MSG_SUBSCRIPTION +from Tribler.community.popularity.payload import SearchResponseItemPayload, TorrentInfoResponsePayload, \ TorrentHealthPayload, ContentSubscription -from Tribler.community.popular.repository import TYPE_TORRENT_HEALTH +from Tribler.community.popularity.repository import TYPE_TORRENT_HEALTH from Tribler.pyipv8.ipv8.test.base import TestBase from Tribler.pyipv8.ipv8.test.mocking.ipv8 import MockIPv8 from Tribler.pyipv8.ipv8.test.util import twisted_wrapper -class TestPopularCommunityBase(TestBase): +class TestPopularityCommunityBase(TestBase): NUM_NODES = 2 def setUp(self): - super(TestPopularCommunityBase, self).setUp() + super(TestPopularityCommunityBase, self).setUp() self.initialize(PopularityCommunity, self.NUM_NODES) def create_node(self, *args, **kwargs): @@ -101,7 +101,7 @@ def update_from_torrent_search_results(self, search_results): pass -class TestPopularCommunity(TestPopularCommunityBase): +class TestPopularityCommunity(TestPopularityCommunityBase): __testing__ = False NUM_NODES = 2 @@ -274,49 +274,6 @@ def test_publish_no_content(self): # Restore logger self.nodes[0].overlay.logger = original_logger - @twisted_wrapper - def test_send_subscription(self): - """ - Tests sending popular content subscription response. - """ - original_logger = self.nodes[0].overlay.logger - self.nodes[0].overlay.logger.error = lambda *args, **kw: self.fake_logger_error(self.nodes[0], *args) - - self.nodes[0].overlay.create_message_packet = lambda _type, _payload: \ - self.fake_create_message_packet(self.nodes[0], _type, _payload) - self.nodes[0].overlay.broadcast_message = lambda packet, peer: \ - self.fake_broadcast_message(self.nodes[0], packet, peer) - - # Two default peers - default_peers = [self.create_node() for _ in range(2)] - # Assuming only one is connected - self.nodes[0].overlay.get_peers = lambda: default_peers[:1] - - # Case1: Try to send subscribe response to non-connected peer - self.nodes[0].unknown_peer_found = False - self.nodes[0].logger_error_called = False - self.nodes[0].overlay.send_subscription_status(default_peers[1], subscribed=True) - yield self.deliver_messages() - - # Expected unknown peer error log - self.assertTrue(self.nodes[0].logger_error_called) - self.assertTrue(self.nodes[0].unknown_peer_found) - - # Case2: Try to send response to the connected peer - self.nodes[0].broadcast_called = False - self.nodes[0].broadcast_packet_type = None - self.nodes[0].overlay.send_subscription_status(default_peers[0], subscribed=True) - yield self.deliver_messages() - - # Expect message to be sent - self.assertTrue(self.nodes[0].packet_created, "Create packet failed") - self.assertEqual(self.nodes[0].packet_type, MSG_SUBSCRIPTION, "Unexpected payload type found") - self.assertTrue(self.nodes[0].broadcast_called, "Should send a message to the peer") - self.assertEqual(self.nodes[0].receiver, default_peers[0], "Intended receiver is different") - - # Restore logger - self.nodes[0].overlay.logger = original_logger - @twisted_wrapper def test_send_torrent_health_response(self): """ @@ -582,8 +539,11 @@ def test_on_subscription_status1(self): Tests receiving subscription status. """ subscribe = True - payload = ContentSubscription(subscribe) + identifier = 123123123 + payload = ContentSubscription(identifier, subscribe) data = self.nodes[1].overlay.create_message_packet(MSG_SUBSCRIPTION, payload) + # Set the cache request + self.nodes[0].overlay.request_cache.pop = lambda prefix, identifer: MockObject() yield self.introduce_nodes() self.assertEqual(len(self.nodes[0].overlay.publishers), 0) @@ -601,9 +561,12 @@ def test_on_subscription_status_with_unsubscribe(self): yield self.introduce_nodes() self.nodes[0].overlay.publishers.add(self.nodes[1].my_peer) self.assertEqual(len(self.nodes[0].overlay.publishers), 1) + # Set the cache request + self.nodes[0].overlay.request_cache.pop = lambda prefix, identifer: MockObject() subscribe = False - payload = ContentSubscription(subscribe) + identifier = 123123123 + payload = ContentSubscription(identifier, subscribe) data = self.nodes[1].overlay.create_message_packet(MSG_SUBSCRIPTION, payload) self.nodes[0].overlay.on_subscription_status(self.nodes[1].my_peer.address, data) @@ -611,12 +574,18 @@ def test_on_subscription_status_with_unsubscribe(self): self.assertEqual(len(self.nodes[0].overlay.publishers), 0) - @twisted_wrapper(5) + @twisted_wrapper def test_search_request_response(self): self.nodes[0].overlay.content_repository = MockRepository() self.nodes[1].overlay.content_repository = MockRepository() self.nodes[1].overlay.publish_latest_torrents = lambda *args, **kwargs: None + def fake_process_torrent_search_response(peer): + peer.called_process_torrent_search_response = True + + self.nodes[0].overlay.process_torrent_search_response = lambda query, payload: \ + fake_process_torrent_search_response(self.nodes[0]) + yield self.introduce_nodes() self.nodes[0].overlay.subscribe_peers() yield self.deliver_messages() @@ -627,6 +596,9 @@ def test_search_request_response(self): yield self.deliver_messages() + self.assertTrue(self.nodes[0].called_process_torrent_search_response) + + @twisted_wrapper def test_send_content_info_request(self): self.nodes[0].overlay.content_repository = MockRepository() diff --git a/Tribler/Test/Community/popular/test_payload.py b/Tribler/Test/Community/popularity/test_payload.py similarity index 95% rename from Tribler/Test/Community/popular/test_payload.py rename to Tribler/Test/Community/popularity/test_payload.py index a7eb316d824..9b7fcc98ec3 100644 --- a/Tribler/Test/Community/popular/test_payload.py +++ b/Tribler/Test/Community/popularity/test_payload.py @@ -2,7 +2,7 @@ import string from unittest import TestCase -from Tribler.community.popular.payload import SearchResponsePayload, SearchResponseItemPayload, ContentInfoRequest, \ +from Tribler.community.popularity.payload import SearchResponsePayload, SearchResponseItemPayload, ContentInfoRequest, \ Pagination, ContentInfoResponse, ContentSubscription, TorrentHealthPayload, ChannelHealthPayload from Tribler.pyipv8.ipv8.messaging.serialization import Serializer @@ -21,13 +21,15 @@ def random_infohash(self): def test_content_subscription(self): """ Test serialization/deserialization of Content subscription """ subscribe = True - subscription = ContentSubscription(subscribe) + identifier = 123123 + subscription = ContentSubscription(identifier, subscribe) serialized = self.serializer.pack_multiple(subscription.to_pack_list()) # Deserialize and test it (deserialized, _) = self.serializer.unpack_multiple(ContentSubscription.format_list, serialized) deserialized_subscription = ContentSubscription.from_unpack_list(*deserialized) + self.assertEqual(deserialized_subscription.identifier, identifier) self.assertTrue(deserialized_subscription.subscribe) def test_torrent_health_payload(self): @@ -169,9 +171,9 @@ def test_content_info_response(self): serialized_response = self.serializer.pack_multiple(in_response.to_pack_list()) # Deserialize request and test it - (deserialized_ressponse, _) = self.serializer.unpack_multiple(ContentInfoResponse.format_list, + (deserialized_response, _) = self.serializer.unpack_multiple(ContentInfoResponse.format_list, serialized_response) - out_request = ContentInfoResponse.from_unpack_list(*deserialized_ressponse) + out_request = ContentInfoResponse.from_unpack_list(*deserialized_response) self.assertEqual(in_response.identifier, out_request.identifier) self.assertEqual(in_response.response, out_request.response) self.assertEqual(in_response.content_type, out_request.content_type) diff --git a/Tribler/Test/Community/popular/test_repository.py b/Tribler/Test/Community/popularity/test_repository.py similarity index 95% rename from Tribler/Test/Community/popular/test_repository.py rename to Tribler/Test/Community/popularity/test_repository.py index 5cb5a03ac69..96ce2c3a7fb 100644 --- a/Tribler/Test/Community/popular/test_repository.py +++ b/Tribler/Test/Community/popularity/test_repository.py @@ -4,8 +4,8 @@ import unittest from Tribler.Test.Core.base_test import MockObject -from Tribler.community.popular.payload import TorrentHealthPayload -from Tribler.community.popular.repository import ContentRepository, DEFAULT_FRESHNESS_LIMIT +from Tribler.community.popularity.payload import TorrentHealthPayload +from Tribler.community.popularity.repository import ContentRepository, DEFAULT_FRESHNESS_LIMIT class TestContentRepository(unittest.TestCase): @@ -129,14 +129,15 @@ def test_update_torrent_info(self): def fake_update_torrent(ref): ref.called_update_torrent = True - self.content_repository.torrent_db.updateTorrent = lambda infohash, **kw: fake_update_torrent(self.content_repository) + self.content_repository.torrent_db.updateTorrent = lambda infohash, **kw: \ + fake_update_torrent(self.content_repository) self.content_repository.has_torrent = lambda infohash: False torrent_info_response = MockObject() torrent_info_response.infohash = 'a' * 20 - torrent_info_response.name = 'ubuntu', + torrent_info_response.name = 'ubuntu' torrent_info_response.length = 123 - torrent_info_response.creation_date =123123123 + torrent_info_response.creation_date = 123123123 torrent_info_response.num_files = 2 torrent_info_response.comment = 'Ubuntu ISO' @@ -217,9 +218,6 @@ def test_search_channel(self): def random_string(size=6, chars=string.ascii_uppercase + string.digits): return ''.join(random.choice(chars) for _ in range(size)) - def random_infohash(): - return ''.join(random.choice('0123456789abcdef') for _ in range(20)) - sample_channels = [] for index in range(10): dbid = index @@ -227,13 +225,14 @@ def random_infohash(): name = random_string() description = random_string(20) nr_torrents = random.randint(1, 10) - nr_favorite = random.randint(1,10) + nr_favorite = random.randint(1, 10) nr_spam = random.randint(1, 10) my_vote = 1 modified = random.randint(1, 10000000) relevance_score = 0.0 - sample_channels.append([dbid, cid, name, description, nr_torrents, nr_favorite, nr_spam, my_vote, modified, relevance_score]) + sample_channels.append([dbid, cid, name, description, nr_torrents, nr_favorite, nr_spam, my_vote, + modified, relevance_score]) def fake_torrentdb_search_channels(_): return sample_channels @@ -289,8 +288,7 @@ def get_torrent(torrent_as_list): 'creation_date': torrent_as_list[5], 'seeders': torrent_as_list[6], 'leechers': torrent_as_list[7], - 'cid': torrent_as_list[8], - } + 'cid': torrent_as_list[8]} def fake_update_torrent(ref): ref.called_update_torrent = True diff --git a/Tribler/Test/Core/Config/test_tribler_config.py b/Tribler/Test/Core/Config/test_tribler_config.py index c4848757098..6ad1d5aafed 100644 --- a/Tribler/Test/Core/Config/test_tribler_config.py +++ b/Tribler/Test/Core/Config/test_tribler_config.py @@ -301,12 +301,12 @@ def test_get_set_methods_preview_channel_community(self): self.tribler_config.set_preview_channel_community_enabled(True) self.assertEqual(self.tribler_config.get_preview_channel_community_enabled(), True) - def test_get_set_methods_popular_community(self): + def test_get_set_methods_popularity_community(self): """ - Check whether popular community get and set methods are working as expected. + Check whether popularity community get and set methods are working as expected. """ - self.tribler_config.set_popular_community_enabled(True) - self.assertEqual(self.tribler_config.get_popular_community_enabled(), True) + self.tribler_config.set_popularity_community_enabled(True) + self.assertEqual(self.tribler_config.get_popularity_community_enabled(), True) def test_get_set_methods_watch_folder(self): """ diff --git a/Tribler/Test/Core/TorrentChecker/test_torrentchecker.py b/Tribler/Test/Core/TorrentChecker/test_torrentchecker.py index 449665979eb..1e326b10886 100644 --- a/Tribler/Test/Core/TorrentChecker/test_torrentchecker.py +++ b/Tribler/Test/Core/TorrentChecker/test_torrentchecker.py @@ -12,7 +12,7 @@ from Tribler.Core.simpledefs import NTFY_TORRENTS from Tribler.Test.Core.base_test import TriblerCoreTest, MockObject from Tribler.Test.twisted_thread import deferred -from Tribler.community.popular.repository import TYPE_TORRENT_HEALTH +from Tribler.community.popularity.repository import TYPE_TORRENT_HEALTH from Tribler.pyipv8.ipv8.util import blocking_call_on_reactor_thread @@ -33,7 +33,7 @@ def setUp(self, annotate=True): self.session.lm.torrent_db = TorrentDBHandler(self.session) self.session.lm.torrent_checker = TorrentChecker(self.session) self.session.lm.tracker_manager = TrackerManager(self.session) - self.session.lm.popular_community = MockObject() + self.session.lm.popularity_community = MockObject() self.torrent_checker = self.session.lm.torrent_checker self.torrent_checker._torrent_db = self.session.open_dbhandler(NTFY_TORRENTS) @@ -190,23 +190,23 @@ def remove_tracker(tracker_url): def test_publish_torrent_result(self): MSG_ZERO_SEED_TORRENT = "Not publishing zero seeded torrents" - MSG_NO_POPULAR_COMMUNITY = "Popular community not available to publish torrent checker result" + MSG_NO_popularity_community = "Popular community not available to publish torrent checker result" def _fake_logger_info(torrent_checker, msg): if msg == MSG_ZERO_SEED_TORRENT: torrent_checker.zero_seed_torrent = True - if msg == MSG_NO_POPULAR_COMMUNITY: - torrent_checker.popular_community_not_found = True + if msg == MSG_NO_popularity_community: + torrent_checker.popularity_community_not_found = True original_logger_info = self.torrent_checker._logger.info self.torrent_checker._logger.info = lambda msg: _fake_logger_info(self.torrent_checker, msg) - def popular_community_queue_content(torrent_checker, _type, _): - torrent_checker.popular_community_queue_content_called = True - torrent_checker.popular_community_queue_content_called_type = _type + def popularity_community_queue_content(torrent_checker, _type, _): + torrent_checker.popularity_community_queue_content_called = True + torrent_checker.popularity_community_queue_content_called_type = _type - self.torrent_checker.tribler_session.lm.popular_community.queue_content = lambda _type, _content: \ - popular_community_queue_content(self.torrent_checker, _type, _content) + self.torrent_checker.tribler_session.lm.popularity_community.queue_content = lambda _type, _content: \ + popularity_community_queue_content(self.torrent_checker, _type, _content) # Case1: Fake torrent checker response, seeders:0 fake_response = {'infohash': 'a'*20, 'seeders': 0, 'leechers': 0, 'last_check': time.time()} @@ -215,17 +215,17 @@ def popular_community_queue_content(torrent_checker, _type, _): # Case2: Positive seeders fake_response['seeders'] = 5 - self.torrent_checker.popular_community_queue_content_called = False - self.torrent_checker.popular_community_queue_content_called_type = None + self.torrent_checker.popularity_community_queue_content_called = False + self.torrent_checker.popularity_community_queue_content_called_type = None self.torrent_checker.publish_torrent_result(fake_response) - self.assertTrue(self.torrent_checker.popular_community_queue_content_called) - self.assertEqual(self.torrent_checker.popular_community_queue_content_called_type, TYPE_TORRENT_HEALTH) + self.assertTrue(self.torrent_checker.popularity_community_queue_content_called) + self.assertEqual(self.torrent_checker.popularity_community_queue_content_called_type, TYPE_TORRENT_HEALTH) # Case3: Popular community is None - self.torrent_checker.tribler_session.lm.popular_community = None + self.torrent_checker.tribler_session.lm.popularity_community = None self.torrent_checker.publish_torrent_result(fake_response) - self.assertTrue(self.torrent_checker.popular_community_not_found) + self.assertTrue(self.torrent_checker.popularity_community_not_found) self.torrent_checker._logger.info = original_logger_info diff --git a/Tribler/Test/test_as_server.py b/Tribler/Test/test_as_server.py index 76d741873c3..964fc51026d 100644 --- a/Tribler/Test/test_as_server.py +++ b/Tribler/Test/test_as_server.py @@ -298,7 +298,7 @@ def setUpPreSession(self): self.config.set_tunnel_community_enabled(False) self.config.set_credit_mining_enabled(False) self.config.set_market_community_enabled(False) - self.config.set_popular_community_enabled(False) + self.config.set_popularity_community_enabled(False) @blocking_call_on_reactor_thread @inlineCallbacks diff --git a/Tribler/community/popular/__init__.py b/Tribler/community/popularity/__init__.py similarity index 100% rename from Tribler/community/popular/__init__.py rename to Tribler/community/popularity/__init__.py diff --git a/Tribler/community/popular/community.py b/Tribler/community/popularity/community.py similarity index 93% rename from Tribler/community/popular/community.py rename to Tribler/community/popularity/community.py index 31368744acd..e65fdc6bcbc 100644 --- a/Tribler/community/popular/community.py +++ b/Tribler/community/popularity/community.py @@ -1,17 +1,17 @@ from twisted.internet.defer import inlineCallbacks from Tribler.Core.simpledefs import SIGNAL_SEARCH_COMMUNITY, SIGNAL_ON_SEARCH_RESULTS -from Tribler.community.popular.constants import MSG_TORRENT_HEALTH_RESPONSE, MSG_CHANNEL_HEALTH_RESPONSE, \ +from Tribler.community.popularity.constants import MSG_TORRENT_HEALTH_RESPONSE, MSG_CHANNEL_HEALTH_RESPONSE, \ MSG_TORRENT_INFO_REQUEST, MSG_TORRENT_INFO_RESPONSE, \ ERROR_UNKNOWN_RESPONSE, MAX_PACKET_PAYLOAD_SIZE, ERROR_UNKNOWN_PEER, ERROR_NO_CONTENT, \ MSG_CONTENT_INFO_REQUEST, \ SEARCH_TORRENT_REQUEST, MSG_CONTENT_INFO_RESPONSE, SEARCH_TORRENT_RESPONSE -from Tribler.community.popular.payload import TorrentHealthPayload, ContentSubscription, TorrentInfoRequestPayload, \ +from Tribler.community.popularity.payload import TorrentHealthPayload, ContentSubscription, TorrentInfoRequestPayload, \ TorrentInfoResponsePayload, SearchResponseItemPayload, \ ContentInfoRequest, Pagination, ContentInfoResponse, decode_values -from Tribler.community.popular.pubsub import PubSubCommunity -from Tribler.community.popular.repository import ContentRepository, TYPE_TORRENT_HEALTH -from Tribler.community.popular.request import SearchRequest +from Tribler.community.popularity.pubsub import PubSubCommunity +from Tribler.community.popularity.repository import ContentRepository, TYPE_TORRENT_HEALTH +from Tribler.community.popularity.request import ContentRequest from Tribler.pyipv8.ipv8.peer import Peer @@ -19,10 +19,10 @@ class PopularityCommunity(PubSubCommunity): """ Community for disseminating the content across the network. Follows publish-subscribe model. """ - MASTER_PUBLIC_KEY = "3081a7301006072a8648ce3d020106052b810400270381920004073beff7002b6a9fc2824a3b1bbb1c4fc32546261" \ - "e3ef7537874560346c5fdc0c17fe654f67d23b08cbb44141879f79a7a4c8deddf9beb4fbc7a0f02ee1586ccebedb6" \ - "23eeef51710108d702f9250361c071482e83c0a4a86c8f45a0b13a19ef83eacb6267b4bfccf220ae5f6d1db7125ea1" \ - "d10da3744b65679828f23376e28b76ab33132b7fa984a77f159dba7351a7" + MASTER_PUBLIC_KEY = "3081a7301006072a8648ce3d020106052b810400270381920004073beff7002b6a9fc2824a3b1bbb1c4fc32546" \ + "261e3ef7537874560346c5fdc0c17fe654f67d23b08cbb44141879f79a7a4c8deddf9beb4fbc7a0f02ee1586cc" \ + "ebedb623eeef51710108d702f9250361c071482e83c0a4a86c8f45a0b13a19ef83eacb6267b4bfccf220ae5f6d" \ + "1db7125ea1d10da3744b65679828f23376e28b76ab33132b7fa984a77f159dba7351a7" master_peer = Peer(MASTER_PUBLIC_KEY.decode('hex')) @@ -219,7 +219,7 @@ def send_content_info_request(self, content_type, request_list, limit=25, peer=N :param limit: Number of expected responses :param peer: Peer to send this request to """ - cache = self.request_cache.add(SearchRequest(self.request_cache, content_type, request_list)) + cache = self.request_cache.add(ContentRequest(self.request_cache, content_type, request_list)) self.logger.info("Sending search request query:%s, identifier:%s", request_list, cache.number) content_request = ContentInfoRequest(cache.number, content_type, request_list, limit) diff --git a/Tribler/community/popular/constants.py b/Tribler/community/popularity/constants.py similarity index 100% rename from Tribler/community/popular/constants.py rename to Tribler/community/popularity/constants.py diff --git a/Tribler/community/popular/payload.py b/Tribler/community/popularity/payload.py similarity index 96% rename from Tribler/community/popular/payload.py rename to Tribler/community/popularity/payload.py index a0f3cdbf9fc..26212303c22 100644 --- a/Tribler/community/popular/payload.py +++ b/Tribler/community/popularity/payload.py @@ -21,20 +21,22 @@ def decode_values(values_str): class ContentSubscription(Payload): - format_list = ['?'] + format_list = ['I', '?'] - def __init__(self, subscribe): + def __init__(self, identifier, subscribe): super(ContentSubscription, self).__init__() + self.identifier = identifier self.subscribe = subscribe def to_pack_list(self): - data = [('?', self.subscribe)] + data = [('I', self.identifier), + ('?', self.subscribe)] return data @classmethod def from_unpack_list(cls, *args): - (subscribe, ) = args - return ContentSubscription(subscribe) + (identifier, subscribe) = args + return ContentSubscription(identifier, subscribe) class TorrentHealthPayload(Payload): @@ -80,7 +82,7 @@ def timestamp(self): class ChannelHealthPayload(Payload): """ - Payload for a channel popularity message in the popular community. + Payload for a channel popularity message in the popularity community. """ format_list = ['varlenI', 'I', 'I', 'I', 'I'] diff --git a/Tribler/community/popular/pubsub.py b/Tribler/community/popularity/pubsub.py similarity index 92% rename from Tribler/community/popular/pubsub.py rename to Tribler/community/popularity/pubsub.py index 4418569d437..d334605a833 100644 --- a/Tribler/community/popular/pubsub.py +++ b/Tribler/community/popularity/pubsub.py @@ -1,18 +1,17 @@ import logging -from Tribler.pyipv8.ipv8.deprecated.community import Community - -from Tribler.pyipv8.ipv8.requestcache import RequestCache +from abc import abstractmethod from copy import copy from twisted.internet.defer import inlineCallbacks from twisted.internet.task import LoopingCall -from Tribler.community.popular.constants import MSG_SUBSCRIPTION, ERROR_UNKNOWN_PEER, MAX_SUBSCRIBERS, MSG_SUBSCRIBE, \ - MAX_PUBLISHERS, PUBLISH_INTERVAL -from Tribler.community.popular.payload import ContentSubscription +from Tribler.community.popularity.constants import MSG_SUBSCRIPTION, ERROR_UNKNOWN_PEER, MAX_SUBSCRIBERS, \ + MSG_SUBSCRIBE, MAX_PUBLISHERS, PUBLISH_INTERVAL +from Tribler.community.popularity.payload import ContentSubscription +from Tribler.community.popularity.request import ContentRequest +from Tribler.pyipv8.ipv8.deprecated.community import Community from Tribler.pyipv8.ipv8.deprecated.payload_headers import BinMemberAuthenticationPayload, GlobalTimeDistributionPayload - from Tribler.pyipv8.ipv8.peer import Peer -from abc import abstractmethod +from Tribler.pyipv8.ipv8.requestcache import RequestCache class PubSubCommunity(Community): @@ -111,6 +110,7 @@ def subscribe(self, peer, subscribe=True): Method to send content subscribe/unsubscribe message. This message is sent to each individual publisher peer we want to subscribe/unsubscribe. """ + cache = self.request_cache.add(ContentRequest(self.request_cache, MSG_SUBSCRIBE, None)) # Add or remove the publisher peer if subscribe: self.publishers.add(peer) @@ -118,7 +118,7 @@ def subscribe(self, peer, subscribe=True): self.publishers.remove(peer) # Create subscription packet and send it - subscription = ContentSubscription(subscribe) + subscription = ContentSubscription(cache.number, subscribe) packet = self.create_message_packet(MSG_SUBSCRIBE, subscription) self.broadcast_message(packet, peer=peer) @@ -144,11 +144,11 @@ def on_subscribe(self, source_address, data): subscribed = False # Send subscription response - self.send_subscription_status(peer, subscribed=subscribed) + self.send_subscription_status(peer, payload.identifier, subscribed=subscribed) return subscribed - def send_subscription_status(self, peer, subscribed=True): + def send_subscription_status(self, peer, identifier, subscribed=True): """ Method to send content subscription message. Content subscription message is send in response to content subscribe or unsubscribe message. @@ -157,7 +157,7 @@ def send_subscription_status(self, peer, subscribed=True): self.logger.error(ERROR_UNKNOWN_PEER) return - subscription = ContentSubscription(subscribed) + subscription = ContentSubscription(identifier, subscribed) packet = self.create_message_packet(MSG_SUBSCRIPTION, subscription) self.broadcast_message(packet, peer=peer) @@ -172,6 +172,10 @@ def on_subscription_status(self, source_address, data): auth, _, payload = self._ez_unpack_auth(ContentSubscription, data) peer = self.get_peer_from_auth(auth, source_address) + cache = self.request_cache.pop(u'request', payload.identifier) + if not cache: + return + if payload.subscribe: self.publishers.add(peer) elif peer in self.publishers: @@ -233,4 +237,3 @@ def pack_sized(self, payload_list, fit_size, start_index=0): def publish_next_content(self): """ Method responsible for publishing content during periodic push """ pass - diff --git a/Tribler/community/popular/repository.py b/Tribler/community/popularity/repository.py similarity index 98% rename from Tribler/community/popular/repository.py rename to Tribler/community/popularity/repository.py index a85866b100e..97711598f34 100644 --- a/Tribler/community/popular/repository.py +++ b/Tribler/community/popularity/repository.py @@ -2,7 +2,7 @@ import time from collections import deque -from Tribler.community.popular.payload import TorrentHealthPayload, SearchResponseItemPayload, ChannelItemPayload +from Tribler.community.popularity.payload import SearchResponseItemPayload, ChannelItemPayload MAX_CACHE = 200 diff --git a/Tribler/community/popular/request.py b/Tribler/community/popularity/request.py similarity index 81% rename from Tribler/community/popular/request.py rename to Tribler/community/popularity/request.py index b4baaae3b20..09ec22f4426 100644 --- a/Tribler/community/popular/request.py +++ b/Tribler/community/popularity/request.py @@ -1,16 +1,15 @@ from twisted.internet.defer import Deferred -from twisted.python.failure import Failure from Tribler.pyipv8.ipv8.requestcache import RandomNumberCache -class SearchRequest(RandomNumberCache): +class ContentRequest(RandomNumberCache): """ This request cache keeps track of all outstanding search requests. """ def __init__(self, request_cache, search_type, query): - super(SearchRequest, self).__init__(request_cache, u"request") + super(ContentRequest, self).__init__(request_cache, u"request") self.query = query self.search_type = search_type self.response = None