Merge branch 'devel' of https://github.com/Tribler/tribler into proto…

…bufserialization_rc

README.rst

@@ -55,7 +55,7 @@ or, if you haven't added your ssh key to your github account:
 
     git clone --recursive https://github.com/Tribler/tribler.git
 
-Second, install the `dependencies <doc/Tribler%20development%20on%20Linux.md>`_.
+Second, install the `dependencies <doc/development/development_on_linux.rst>`_.
 
 Done!
 Now you can run tribler by executing the ``tribler.sh`` script on the root of the repository:

Tribler/Core/CacheDB/SqliteCacheDBHandler.py

@@ -1872,6 +1872,16 @@ def getChannelTorrents(self, infohash, keys):
 
         return self.__fixTorrents(keys, results)
 
+    def get_random_channel_torrents(self, keys, limit=10):
+        """
+        Return some random (channel) torrents from the database.
+        """
+        sql = "SELECT %s FROM ChannelTorrents, Torrent " \
+              "WHERE ChannelTorrents.torrent_id = Torrent.torrent_id AND Torrent.name IS NOT NULL " \
+              "ORDER BY RANDOM() LIMIT ?" % ", ".join(keys)
+        results = self._db.fetchall(sql, (limit,))
+        return self.__fixTorrents(keys, results)
+
     def getTorrentFromChannelTorrentId(self, channeltorrent_id, keys):
         sql = "SELECT " + ", ".join(keys) + """ FROM Torrent, ChannelTorrents
               WHERE Torrent.torrent_id = ChannelTorrents.torrent_id AND ChannelTorrents.id = ?"""

Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py

@@ -26,7 +26,7 @@
 if sys.platform == "win32":
     try:
         import ctypes
-    except:
+    except ImportError:
         pass
 
 
@@ -684,7 +684,7 @@ def set_selected_files(self, selected_files=None):
                             if sys.platform == "win32":
                                 ctypes.windll.kernel32.SetFileAttributesW(
                                     unwanteddir_abs, 2)  # 2 = FILE_ATTRIBUTE_HIDDEN
-                        except:
+                        except OSError:
                             self._logger.error("LibtorrentDownloadImpl: could not create %s" % unwanteddir_abs)
                             # Note: If the destination directory can't be accessed, libtorrent will not be able to store the files.
                             # This will result in a DLSTATUS_STOPPED_ON_ERROR.

Tribler/Core/Modules/restapi/channels/base_channels_endpoint.py

@@ -4,7 +4,7 @@
 from twisted.web import http, resource
 from Tribler.Core.simpledefs import NTFY_CHANNELCAST
 from Tribler.community.allchannel.community import AllChannelCommunity
-
+from Tribler.dispersy.exception import CommunityNotFoundException
 
 UNKNOWN_CHANNEL_RESPONSE_MSG = "the channel with the provided cid is not known"
 UNAUTHORIZED_RESPONSE_MSG = "you are not authorized to perform this request"
@@ -72,3 +72,14 @@ def vote_for_channel(self, cid, vote):
             if isinstance(community, AllChannelCommunity):
                 community.disp_create_votecast(cid, vote, int(time.time()))
                 break
+
+    def get_community_for_channel_id(self, channel_id):
+        """
+        Returns a Dispersy community from the given channel id. The Community object can be used to delete/add torrents
+        or modify playlists in a specific channel.
+        """
+        dispersy_cid = str(self.channel_db_handler.getDispersyCIDFromChannelId(channel_id))
+        try:
+            return self.session.get_dispersy_instance().get_community(dispersy_cid)
+        except CommunityNotFoundException:
+            return None

Tribler/Core/Modules/restapi/channels/channels_endpoint.py

@@ -1,5 +1,6 @@
 from Tribler.Core.Modules.restapi.channels.base_channels_endpoint import BaseChannelsEndpoint
 from Tribler.Core.Modules.restapi.channels.channels_discovered_endpoint import ChannelsDiscoveredEndpoint
+from Tribler.Core.Modules.restapi.channels.channels_popular_endpoint import ChannelsPopularEndpoint
 from Tribler.Core.Modules.restapi.channels.channels_subscription_endpoint import ChannelsSubscribedEndpoint
 
 
@@ -11,6 +12,7 @@ class ChannelsEndpoint(BaseChannelsEndpoint):
     def __init__(self, session):
         BaseChannelsEndpoint.__init__(self, session)
 
-        child_handler_dict = {"subscribed": ChannelsSubscribedEndpoint, "discovered": ChannelsDiscoveredEndpoint}
+        child_handler_dict = {"subscribed": ChannelsSubscribedEndpoint, "discovered": ChannelsDiscoveredEndpoint,
+                              "popular": ChannelsPopularEndpoint}
         for path, child_cls in child_handler_dict.iteritems():
             self.putChild(path, child_cls(self.session))

Tribler/Core/Modules/restapi/channels/channels_playlists_endpoint.py

@@ -1,6 +1,7 @@
 import json
 from Tribler.Core.CacheDB.sqlitecachedb import str2bin
 from Tribler.Core.Modules.restapi.channels.base_channels_endpoint import BaseChannelsEndpoint
+from Tribler.Core.Modules.restapi.util import convert_db_torrent_to_json
 
 
 class ChannelsPlaylistsEndpoint(BaseChannelsEndpoint):
@@ -33,8 +34,14 @@ def render_GET(self, request):
                         "name": "My first playlist",
                         "description": "Funny movies",
                         "torrents": [{
-                            "name": "movie_1",
-                            "infohash": "e940a7a57294e4c98f62514b32611e38181b6cae"
+                            "id": 4,
+                            "infohash": "97d2d8f5d37e56cfaeaae151d55f05b077074779",
+                            "name": "Ubuntu-16.04-desktop-amd64",
+                            "size": 8592385,
+                            "category": "other",
+                            "num_seeders": 42,
+                            "num_leechers": 184,
+                            "last_tracker_check": 1463176959
                         }, ... ]
                     }, ...]
                 }
@@ -48,12 +55,13 @@ def render_GET(self, request):
 
         playlists = []
         req_columns = ['Playlists.id', 'Playlists.name', 'Playlists.description']
-        req_columns_torrents = ['ChannelTorrents.name', 'Torrent.infohash']
+        req_columns_torrents = ['Torrent.torrent_id', 'infohash', 'Torrent.name', 'length', 'Torrent.category',
+                                'num_seeders', 'num_leechers', 'last_tracker_check', 'ChannelTorrents.inserted']
         for playlist in self.channel_db_handler.getPlaylistsFromChannelId(channel[0], req_columns):
             # Fetch torrents in the playlist
-            torrents = []
-            for torrent in self.channel_db_handler.getTorrentsFromPlaylist(playlist[0], req_columns_torrents):
-                torrents.append({"name": torrent[0], "infohash": str2bin(torrent[1]).encode('hex')})
+            playlist_torrents = self.channel_db_handler.getTorrentsFromPlaylist(playlist[0], req_columns_torrents)
+            torrents = [convert_db_torrent_to_json(torrent_result) for torrent_result in playlist_torrents
+                        if torrent_result[2] is not None]
 
             playlists.append({"id": playlist[0], "name": playlist[1], "description": playlist[2], "torrents": torrents})
 

Tribler/Core/Modules/restapi/channels/channels_popular_endpoint.py

@@ -0,0 +1,51 @@
+import json
+from twisted.web import http
+from Tribler.Core.Modules.restapi.channels.base_channels_endpoint import BaseChannelsEndpoint
+from Tribler.Core.Modules.restapi.util import convert_db_channel_to_json
+
+
+class ChannelsPopularEndpoint(BaseChannelsEndpoint):
+
+    def render_GET(self, request):
+        """
+        .. http:get:: /channels/popular?limit=(int:max nr of channels)
+
+        A GET request to this endpoint will return the most popular discovered channels in Tribler.
+        You can optionally pass a limit parameter to limit the number of results.
+
+            **Example request**:
+
+            .. sourcecode:: none
+
+                curl -X GET http://localhost:8085/channels/popular?limit=1
+
+            **Example response**:
+
+            .. sourcecode:: javascript
+
+                {
+                    "channels": [{
+                        "id": 3,
+                        "dispersy_cid": "da69aaad39ccf468aba2ab9177d5f8d8160135e6",
+                        "name": "My fancy channel",
+                        "description": "A description of this fancy channel",
+                        "subscribed": False,
+                        "votes": 23,
+                        "torrents": 3,
+                        "spam": 5,
+                        "modified": 14598395,
+                    }]
+                }
+        """
+        limit_channels = 10
+
+        if 'limit' in request.args and len(request.args['limit']) > 0:
+            limit_channels = int(request.args['limit'][0])
+
+            if limit_channels <= 0:
+                request.setResponseCode(http.BAD_REQUEST)
+                return json.dumps({"error": "the limit parameter must be a positive number"})
+
+        popular_channels = self.channel_db_handler.getMostPopularChannels(max_nr=limit_channels)
+        results_json = [convert_db_channel_to_json(channel) for channel in popular_channels]
+        return json.dumps({"channels": results_json})

Tribler/Core/Modules/restapi/channels/channels_torrents_endpoint.py

@@ -8,6 +8,10 @@
 from Tribler.Core.exceptions import DuplicateTorrentFileError
 
 
+UNKNOWN_TORRENT_MSG = "this torrent is not found in the specified channel"
+UNKNOWN_COMMUNITY_MSG = "the community for the specified channel cannot be found"
+
+
 class ChannelsTorrentsEndpoint(BaseChannelsEndpoint):
     """
     This class is responsible for managing requests regarding torrents in a channel.
@@ -182,3 +186,46 @@ def on_receive_magnet_meta_info(meta_info):
             return BaseChannelsEndpoint.return_500(self, request, ex)
 
         return json.dumps({"added": self.infohash})
+
+    def render_DELETE(self, request):
+        """
+        .. http:delete:: /channels/discovered/(string: channelid)/torrents/(string: torrent infohash)
+
+        Remove a torrent with a given infohash from a given channel.
+
+            **Example request**:
+
+            .. sourcecode:: none
+
+                curl -X DELETE http://localhost:8085/channels/discovered/abcdefg/torrents/
+                97d2d8f5d37e56cfaeaae151d55f05b077074779
+
+            **Example response**:
+
+            .. sourcecode:: javascript
+
+                {
+                    "removed": True
+                }
+
+            :statuscode 404: if the channel is not found or if the torrent is not found in the specified channel
+        """
+        channel_info = self.get_channel_from_db(self.cid)
+        if channel_info is None:
+            return ChannelsTorrentsEndpoint.return_404(request)
+
+        torrent_db_columns = ['Torrent.torrent_id', 'infohash', 'Torrent.name', 'length', 'Torrent.category',
+                              'num_seeders', 'num_leechers', 'last_tracker_check', 'ChannelTorrents.dispersy_id']
+        torrent_info = self.channel_db_handler.getTorrentFromChannelId(channel_info[0], self.infohash.decode('hex'),
+                                                                       torrent_db_columns)
+
+        if torrent_info is None:
+            return BaseChannelsEndpoint.return_404(request, message=UNKNOWN_TORRENT_MSG)
+
+        channel_community = self.get_community_for_channel_id(channel_info[0])
+        if channel_community is None:
+            return BaseChannelsEndpoint.return_404(request, message=UNKNOWN_COMMUNITY_MSG)
+
+        channel_community.remove_torrents([torrent_info[8]])  # the 8th index is the dispersy id of the channel torrent
+
+        return json.dumps({"removed": True})

Tribler/Core/Modules/restapi/root_endpoint.py

@@ -6,6 +6,7 @@
 from Tribler.Core.Modules.restapi.my_channel_endpoint import MyChannelEndpoint
 from Tribler.Core.Modules.restapi.search_endpoint import SearchEndpoint
 from Tribler.Core.Modules.restapi.settings_endpoint import SettingsEndpoint
+from Tribler.Core.Modules.restapi.torrents_endpoint import TorrentsEndpoint
 from Tribler.Core.Modules.restapi.variables_endpoint import VariablesEndpoint
 from Tribler.Core.Modules.restapi.create_torrent_endpoint import CreateTorrentEndpoint
 
@@ -33,7 +34,8 @@ def start_endpoints(self):
         """
         child_handler_dict = {"search": SearchEndpoint, "channels": ChannelsEndpoint, "mychannel": MyChannelEndpoint,
                               "settings": SettingsEndpoint, "variables": VariablesEndpoint,
-                              "downloads": DownloadsEndpoint, "createtorrent": CreateTorrentEndpoint}
+                              "downloads": DownloadsEndpoint, "createtorrent": CreateTorrentEndpoint,
+                              "torrents": TorrentsEndpoint}
 
         for path, child_cls in child_handler_dict.iteritems():
             self.putChild(path, child_cls(self.session))

Tribler/Core/Modules/restapi/torrents_endpoint.py

@@ -0,0 +1,71 @@
+import json
+
+from twisted.web import http, resource
+
+from Tribler.Core.Modules.restapi.util import convert_db_torrent_to_json
+from Tribler.Core.simpledefs import NTFY_TORRENTS, NTFY_CHANNELCAST
+
+
+class TorrentsEndpoint(resource.Resource):
+
+    def __init__(self, session):
+        resource.Resource.__init__(self)
+
+        child_handler_dict = {"random": TorrentsRandomEndpoint}
+
+        for path, child_cls in child_handler_dict.iteritems():
+            self.putChild(path, child_cls(session))
+
+
+class TorrentsRandomEndpoint(resource.Resource):
+
+    def __init__(self, session):
+        resource.Resource.__init__(self)
+        self.session = session
+        self.channel_db_handler = self.session.open_dbhandler(NTFY_CHANNELCAST)
+        self.torrents_db_handler = self.session.open_dbhandler(NTFY_TORRENTS)
+
+    def render_GET(self, request):
+        """
+        .. http:get:: /torrents/random?limit=(int: max nr of torrents)
+
+        A GET request to this endpoint returns random (channel) torrents.
+        You can optionally specify a limit parameter to limit the maximum number of results. By default, this is 10.
+
+            **Example request**:
+
+            .. sourcecode:: none
+
+                curl -X GET http://localhost:8085/torrents/random?limit=1
+
+            **Example response**:
+
+            .. sourcecode:: javascript
+
+                {
+                    "torrents": [{
+                        "id": 4,
+                        "infohash": "97d2d8f5d37e56cfaeaae151d55f05b077074779",
+                        "name": "Ubuntu-16.04-desktop-amd64",
+                        "size": 8592385,
+                        "category": "other",
+                        "num_seeders": 42,
+                        "num_leechers": 184,
+                        "last_tracker_check": 1463176959
+                    }]
+                }
+        """
+        limit_torrents = 10
+
+        if 'limit' in request.args and len(request.args['limit']) > 0:
+            limit_torrents = int(request.args['limit'][0])
+
+            if limit_torrents <= 0:
+                request.setResponseCode(http.BAD_REQUEST)
+                return json.dumps({"error": "the limit parameter must be a positive number"})
+
+        torrent_db_columns = ['Torrent.torrent_id', 'infohash', 'Torrent.name', 'length', 'Torrent.category',
+                              'num_seeders', 'num_leechers', 'last_tracker_check', 'ChannelTorrents.inserted']
+        popular_torrents = self.channel_db_handler.get_random_channel_torrents(torrent_db_columns, limit=limit_torrents)
+        results_json = [convert_db_torrent_to_json(popular_torrent) for popular_torrent in popular_torrents]
+        return json.dumps({"torrents": results_json})

Tribler/Core/Utilities/twisted_utils.py

@@ -53,7 +53,8 @@ def isInThreadPool():
     """
     Check if we are currently on one of twisted threadpool threads.
     """
-    return current_thread() in reactor.threadpool.threads
+    threadpool = reactor.getThreadPool()
+    return threadpool is not None and current_thread() in threadpool.threads
 
 
 #

Tribler/Core/osutils.py

@@ -133,16 +133,16 @@ def get_desktop_dir():
 elif is_android():
 
     def get_home_dir():
-        return os.path.realpath(os.environ['ANDROID_PRIVATE'])
+        return os.path.realpath(unicode(os.environ['ANDROID_PRIVATE']))
 
     def get_appstate_dir():
-        return os.path.join(get_home_dir(), '.Tribler')
+        return os.path.join(get_home_dir(), u'.Tribler')
 
     def get_picture_dir():
-        return os.path.join(get_desktop_dir(), 'DCIM')
+        return os.path.join(get_desktop_dir(), u'DCIM')
 
     def get_desktop_dir():
-        return os.path.realpath(os.environ['EXTERNAL_STORAGE'])
+        return os.path.realpath(unicode(os.environ['EXTERNAL_STORAGE']))
 
 else:
     # linux or darwin (mac)

Tribler/Main/Dialogs/systray.py

@@ -1,5 +1,6 @@
 # Author : Choopan RATTANAPOKA, Jie Yang, Arno Bakker
 # see LICENSE.txt for license information
+import sys
 import wx
 from traceback import print_exc
 from Tribler.Main.Utility.utility import speed_format
@@ -42,7 +43,11 @@ def updateTooltip(self, download_speed=0, upload_speed=0):
     def updateIcon(self, iconifying=False):
         remove = True
 
-        mintray = self.utility.read_config('mintray')
+        if sys.platform == 'win32':
+            mintray = 2
+        else:
+            mintray = self.utility.read_config('mintray')
+
         if (mintray >= 2) or ((mintray >= 1) and iconifying):
             remove = False
 

Tribler/Main/Utility/GuiDBHandler.py

@@ -307,7 +307,7 @@ def startWorker(
                 filename, line, function, _ = extract_stack(limit=2)[0]
                 _, filename = os.path.split(filename)
                 jobID = u"%s:%s (%s)" % (filename, line, function)
-            except:
+            except IndexError:
                 pass
 
     result = ASyncDelayedResult(jobID)