reduce creditmining load at Tribler startup - prioritize other things…

… first

Tribler/Main/tribler_main.py

@@ -27,6 +27,7 @@
 import tempfile
 import traceback
 import urllib2
+import shutil
 from collections import defaultdict
 from random import randint
 from traceback import print_exc
@@ -113,7 +114,7 @@ def __init__(self, params, installdir, autoload_discovery=True,
 
         self.webUI = None
         self.utility = None
-
+        self.boosting_manager = None
         # Stage 1 start
         session = self.InitStage1(installdir, autoload_discovery=autoload_discovery,
                                   use_torrent_search=use_torrent_search, use_channel_search=use_channel_search)
@@ -271,8 +272,6 @@ def __init__(self, params, installdir, autoload_discovery=True,
             # gracefully closes Tribler after 120 seconds.
             # wx.CallLater(120*1000, wx.GetApp().Exit)
 
-            self.boosting_manager = BoostingManager.get_instance()
-
             self.ready = True
 
         except Exception as e:
@@ -419,6 +418,8 @@ def PostInit2(self):
         # TODO(emilon): Use the LogObserver I already implemented
         # self.dispersy.callback.attach_exception_handler(self.frame.exceptionHandler)
 
+
+        self.boosting_manager = BoostingManager.get_instance()
         startWorker(None, self.loadSessionCheckpoint, delay=5.0, workerType="ThreadPool")
 
     def startAPI(self, session):
@@ -911,9 +912,8 @@ def OnExit(self):
 
         if self.boosting_manager:
 
-            #remove credit mining data
-            #TODO(ardhi) : not persistent mode only
-            import shutil; shutil.rmtree(self.boosting_manager.credit_mining_path, ignore_errors=True)
+            #remove credit mining data in not persistent mode
+            shutil.rmtree(self.boosting_manager.credit_mining_path, ignore_errors=True)
 
             self.boosting_manager.shutdown()
             self.boosting_manager.del_instance()

Tribler/Main/vwxGUI/CreditMiningPanel.py

@@ -234,7 +234,7 @@ def __init__(self, parent):
         self.utility = self.guiutility.utility
         self.installdir = self.utility.getPath()
 
-        self.boosting_manager = BoostingManager.get_instance()
+        self.boosting_manager = BoostingManager.get_instance(self.utility.session)
 
         self.tdb = self.utility.session.open_dbhandler(NTFY_TORRENTS)
 
@@ -250,6 +250,7 @@ def __init__(self, parent):
             self.main_sizer.Add(self.header, 0, wx.EXPAND)
 
         self.main_splitter = wx.SplitterWindow(self, style=wx.SP_BORDER)
+        self.main_splitter.SetMinimumPaneSize(300)
 
         self.sourcelist = CpanelCheckListCtrl(self.main_splitter, -1,
                                 agwStyle=wx.LC_REPORT | wx.LC_NO_HEADER | wx.LC_VRULES
@@ -325,11 +326,10 @@ def AddComponents(self,parent):
         self.infoPanel.SetSizer(if_sizer)
 
         self.sourcelist.Hide()
-        self.loading_holder =  wx.StaticText(self.main_splitter, -1, 'Loading..')
+        self.loading_holder = wx.StaticText(self.main_splitter, -1, 'Loading..')
 
-        parent.SplitVertically(self.loading_holder, self.infoPanel)
-        parent.SetMinimumPaneSize(100)
-        parent.SetSashGravity(0.25)
+        parent.SplitVertically(self.loading_holder, self.infoPanel,1)
+        parent.SetSashGravity(0.3)
         self.main_sizer.Add(parent, 1, wx.EXPAND)
 
     def OnItemSelected(self, event):

Tribler/Main/vwxGUI/home.py

@@ -51,7 +51,7 @@ def __init__(self, parent):
         self.guiutility = GUIUtility.getInstance()
         self.gui_image_manager = GuiImageManager.getInstance()
         self.session = self.guiutility.utility.session
-        self.boosting_manager = BoostingManager.get_instance(self.session)
+        self.boosting_manager = None
 
         #dispersy_cid:Channel
         self.channels = {}
@@ -108,7 +108,7 @@ def __init__(self, parent):
         vSizer.Add(hSizer, 0, wx.ALIGN_CENTER)
         vSizer.AddStretchSpacer()
 
-        # channel panel is popular channel
+        # channel panel is for popular channel
         self.channel_panel = ScrolledPanel(self, 1)
         self.channel_panel.SetBackgroundColour(wx.WHITE)
         self.channel_panel.SetForegroundColour(parent.GetForegroundColour())
@@ -209,6 +209,9 @@ def CreateChannelItem(self, parent, channel, torrents, max_fav):
         from Tribler.Main.Utility.GuiDBTuples import Channel as ChannelObj
         assert isinstance(channel, ChannelObj), "Type channel should be ChannelObj %s" %channel
 
+        if not self.boosting_manager:
+            self.boosting_manager = BoostingManager.get_instance(self.session)
+
         STRING_LENGTH = 35
 
         vsizer = wx.BoxSizer(wx.VERTICAL)

Tribler/Policies/BoostingManager.py

@@ -172,18 +172,15 @@ def __init__(self, session, utility=None, policy=SeederRatioPolicy, src_interval
         self.utility = utility
         self.credit_mining_path = os.path.join(DefaultDownloadStartupConfig.getInstance().get_dest_dir(), "credit_mining")
 
-        try:
-            if not os.path.exists(self.credit_mining_path):
-                os.mkdir(self.credit_mining_path)
-        except:
-            pass
+        if not os.path.exists(self.credit_mining_path):
+            os.mkdir(self.credit_mining_path)
 
         self.boosting_sources = {}
         self.torrents = {}
         self.policy = None
         self.share_mode_target = 3
 
-        # TODO(ardhi) : hardcode some of the interval for now
+        # change some params here, if you want
         self.max_torrents_per_source = max_per_source
         self.max_torrents_active = max_active
         self.source_interval = src_interval
@@ -192,17 +189,17 @@ def __init__(self, session, utility=None, policy=SeederRatioPolicy, src_interval
         self.policy = policy(self.session)
         self.tracker_interval = 300
         self.initial_tracker_interval = 25
-        self.logging_interval = 60
+        self.logging_interval = 40
         self.initial_logging_interval = 35
 
-        self.load_config()
-
         self.set_share_mode_params(share_mode_target=self.share_mode_target)
 
         if os.path.exists(config_file):
             logger.info("Config file %s", open(config_file).read())
+            self.load_config()
         else:
-            logger.info("Config file missing")
+            logger.warning("Initial config file missing")
+
 
         # TODO(emilon): Refactor this to use taskmanager
         self.session.lm.threadpool.add_task(self._select_torrent, self.initial_swarm_interval)
@@ -231,8 +228,11 @@ def shutdown(self):
             except:
                 continue
 
-    def get_source_object(self, sourcekey):
 
+    def get_source_object(self, sourcekey):
+        """
+        Get the actual object of the source string/key
+        """
         # first, assume it's already source-friendly format
         ret = self.boosting_sources.get(sourcekey, None)
 
@@ -249,17 +249,27 @@ def get_source_object(self, sourcekey):
 
 
     def set_enable_mining(self, source, mining_bool=True, force_restart=False):
+        """
+        Dynamically enable/disable mining source.
+        :param source: source, perhaps a url, byte-channelid, or directory
+        :param mining_bool: enable/disable
+        :param force_restart: do we really need to restart the mining?
+        """
+
+        # Flag : there are not any swarm stored for this source
         tor_not_exist = True
 
         for ihash, tor in self.torrents.iteritems():
             if tor['source'] == source:
                 tor_not_exist = False
                 self.torrents[ihash]['enabled'] = mining_bool
 
+                # pause torrent download from disabled source
                 if (not mining_bool):
                     self.stop_download(tor)
 
-        # this only happen via new channel boosting interface
+        # this only happen via new channel boosting interface. (CreditMiningPanel)
+        # case : just start mining a particular source (e.g. PreviewChannel)
         if tor_not_exist and mining_bool and not (source in self.boosting_sources.keys()):
             self.add_source(source)
             self.set_archive(source, False)
@@ -305,12 +315,12 @@ def add_source(self, source):
             args = (self.session, self.session.lm.threadpool, source, self.source_interval, self.max_torrents_per_source, self.on_torrent_insert)
                 # pylint: disable=star-args
 
+
             try:
                 isdir = os.path.isdir(source)
             except TypeError:
                 isdir = False
 
-
             if isdir:
                 self.boosting_sources[source] = DirectorySource(*args)
             elif source.startswith('http://') or source.startswith('https://'):
@@ -339,6 +349,10 @@ def remove_source(self, source_key):
             logger.info("Removing from possible swarms")
 
     def compare_torrents(self, t1, t2):
+        """
+        comparing swarms. We don't want to download same swarm with different infohash
+        :return: whether those t1 and t2 similar enough
+        """
         # pylint: disable=no-self-use, bad-builtin
         try:
             ff = lambda ft: ft[1] > 1024 * 1024
@@ -356,8 +370,14 @@ def compare_torrents(self, t1, t2):
             return False
 
     def on_torrent_insert(self, source, infohash, torrent):
-        # Remember where we got this torrent from
-
+        """
+        This function called when a source finally determined. Fetch some torrents from it,
+        then insert it to our data
+        :param source:
+        :param infohash: torrent infohash
+        :param torrent: torrent object (dictionary format)
+        :return:
+        """
         try:
             isdir = os.path.isdir(source)
         except TypeError:
@@ -369,6 +389,8 @@ def on_torrent_insert(self, source, infohash, torrent):
             source_str = source.encode('hex')
         else:
             source_str = 'unknown source'
+
+        # Remember where we got this torrent from
         torrent['source'] = source_str
 
         boost_source = self.boosting_sources.get(source, None)
@@ -387,6 +409,7 @@ def on_torrent_insert(self, source, infohash, torrent):
             else:
                 self._logger.info("Not collected yet: %s %s ", infohash, torrent['name'])
                 # TODO(emilon): Handle the case where the torrent hasn't been collected. (collected from the DHT)
+                # ardhi : so far, this case won't happen because torrent already defined in _update in BoostingSource
                 # torrent['metainfo'] = TorrentDefNoMetainfo(infohash, torrent['name'])
                 pass
 
@@ -402,17 +425,26 @@ def on_torrent_insert(self, source, infohash, torrent):
                     self.stop_download(duplicate)
 
         self.torrents[infohash] = torrent
-        # logger.info("Got new torrent %s from %s", infohash.encode('hex'),
-        #             source_str)
 
     def _check_main_download(self, idx_multiplier):
-
+        """
+        check the main downloading activity by user.
+        :param idx_multiplier: it's 'level' of how severe the downloading activity will affect credit mining
+        The level consist of 6 level, from 0-6.
+        0 : main downloading activity won't change credit mining speed
+        1 : main downloading activity change credit mining speed a bit
+        6 : main downloading activity change credit mining speed significantly
+        :return: (#active mining swarm, TOTAL assigned priority for all active swarm in credit mining)
+        """
+
+        # find the total priority of CM active downloading
         total_cm_prio = sum([tr['download'].handle.status().priority or 0
                        for tr in self.torrents.values() if tr and tr.get('download', False) and
                              tr['download'].handle])
 
         (hits_num, hits) = self.gui_util.library_manager.getHitsInCategory()
 
+        # total priority of main downloading
         total_main_prio = sum([chn_tor.ds.get_download().handle.status().priority or 0
              for chn_tor in hits if chn_tor and chn_tor.ds and chn_tor.ds.get_download().handle])
 
@@ -435,7 +467,6 @@ def scrape_trackers(self):
         # 1 is the lowest priority we'd want to assign
         new_prio = (total_prio_tor/float(num_dl) if num_dl else self.DEFAULT_PRIORITY_TORRENT) or 1
 
-
         for infohash, torrent in self.torrents.iteritems():
             tf = torrent['metainfo']
 
@@ -479,8 +510,8 @@ def _logtorrentpeer(status, lt_torrent):
                         print out
 
                     print "----------------------"
-                # _logtorrentpeer(status, lt_torrent)
 
+                _logtorrentpeer(status, lt_torrent)
 
                 # change priority (reduce load on main downloading)
                 if new_prio and int(new_prio) != status.priority:
@@ -529,8 +560,7 @@ def do_start():
             dscfg.set_dest_dir(self.credit_mining_path)
             dscfg.set_safe_seeding(False)
 
-
-            #TODO (ardhi) debug variable so I can change this later
+            # just a debug variable
             tobj = torrent
 
             preload = tobj.get('preload', False)
@@ -803,7 +833,6 @@ def get_channel_id():
             logger.info("Channel %s was not ready, waits for next interval (%d chn)", hexlify(self.source), len(dispersy.get_communities()))
             self.session.lm.threadpool.add_task(lambda cid=self.source: self._load(cid), 10, task_name=self.source)
 
-
     def _check_tor(self):
         from Tribler.Main.Utility.GuiDBHandler import startWorker
         def doGui(delayedResult):