properly remove credit mining data and sources

Conflicts:
	Tribler/Main/tribler_main.py
	Tribler/Main/vwxGUI/list.py

Tribler/Main/tribler_main.py

@@ -865,6 +865,15 @@ def OnExit(self):
             self.webUI.stop()
             self.webUI.delInstance()
 
+        if self.boosting_manager:
+
+            #remove credit mining data
+            #TODO(ardhi) : not persistent mode only
+            import shutil; shutil.rmtree(self.boosting_manager.credit_mining_path)
+
+            self.boosting_manager.shutdown()
+            self.boosting_manager.del_instance()
+
         if self.frame:
             self.frame.Destroy()
             self.frame = None

Tribler/Main/vwxGUI/list.py

@@ -302,6 +302,70 @@ def downloadStarted(self, _):
         self.refresh()
 
 
+class CreditMiningSearchManager(BaseManager):
+
+    def __init__(self, list):
+        BaseManager.__init__(self, list)
+        self.boosting_manager = BoostingManager.get_instance()
+        self.library_manager = self.guiutility.library_manager
+
+    def refresh(self):
+        startWorker(self._on_data, self.getHitsInCategory, uId=u"CreditMiningSearchManager_refresh", retryOnBusy=True, priority=GUI_PRI_DISPERSY)
+
+    def getTorrentFromInfohash(self, infohash):
+        torrent = self.boosting_manager.torrents.get(infohash, None)
+        if torrent:
+            t = LibraryTorrent('', infohash, name=torrent['name'], length=torrent['length'], category='', status='',
+                               num_seeders=torrent['num_seeders'], num_leechers=torrent['num_leechers'])
+            t.torrent_db = self.library_manager.torrent_db
+            t.channelcast_db = self.library_manager.channelcast_db
+            # t.channel
+            self.library_manager.addDownloadState(t)
+            return t
+
+    def getHitsInCategory(self):
+        hits = [self.getTorrentFromInfohash(infohash) for infohash in self.boosting_manager.torrents.keys()]
+        return [len(hits), hits]
+
+    def refresh_partial(self, ids):
+        for infohash in ids:
+            startWorker(self.list.RefreshDelayedData, self.getTorrentFromInfohash, cargs=(infohash,), wargs=(infohash,), retryOnBusy=True, priority=GUI_PRI_DISPERSY)
+
+    def refresh_if_exists(self, infohashes, force=False):
+        if any([self.boosting_manager.torrents.has_key(infohash) for infohash in infohashes]):
+            print >> sys.stderr, long(time()), "Scheduling a refresh, missing some infohashes in the Credit Mining overview"
+            self.refresh()
+        else:
+            print >> sys.stderr, long(time()), "Not scheduling a refresh"
+
+    def refresh_or_expand(self, infohash):
+        if not self.list.InList(infohash):
+            def select(delayedResult):
+                delayedResult.get()
+                self.refresh_or_expand(infohash)
+
+            startWorker(select, self.refresh_partial, wargs=([infohash],), priority=GUI_PRI_DISPERSY)
+        else:
+            self.list.Select(infohash)
+
+    @forceWxThread
+    def _on_data(self, delayedResult):
+        total_items, data = delayedResult.get()
+        self.list.SetData(data)
+        self.list.Layout()
+
+    def torrentUpdated(self, infohash):
+        if self.list.InList(infohash):
+            self.do_or_schedule_partial([infohash])
+
+    def torrentsUpdated(self, infohashes):
+        infohashes = [infohash for infohash in infohashes if self.list.InList(infohash)]
+        self.do_or_schedule_partial(infohashes)
+
+    def downloadStarted(self, infohash):
+        self.refresh()
+
+
 class ChannelSearchManager(BaseManager):
 
     def __init__(self, list):
@@ -1945,6 +2009,232 @@ def GetFilterMessage(self, empty=False):
         return header, message
 
 
+class CreditMiningList(SizeList):
+
+    def __init__(self, parent):
+        self.boosting_manager = BoostingManager.get_instance()
+        self.guiutility = GUIUtility.getInstance()
+        self.utility = self.guiutility.utility
+
+        self.statefilter = None
+        self.newfilter = False
+        self.prevStates = {}
+        self.oldDS = {}
+
+        self.initnumitems = False
+
+        columns = [{'name': 'Speed up/down', 'width': '32em', 'autoRefresh': False},
+                   {'name': 'Bytes up/down', 'width': '32em', 'autoRefresh': False},
+                   {'name': 'Seeders/leechers', 'width': '27em', 'autoRefresh': False},
+                   {'name': 'Duplicate', 'showColumname': False, 'width':  '2em'},
+                   {'name': 'Hash', 'width':  '27em', 'fmt': lambda ih: ih.encode('hex')[:10]},
+                   {'name': 'Source', 'width': '40em', 'type': 'method', 'method': self.CreateSource},
+                   {'name': 'Investment status', 'width': '32em', 'autoRefresh': False}]
+
+        columns = self.guiutility.SetColumnInfo(CreditMiningListItem, columns)
+        ColumnsManager.getInstance().setColumns(CreditMiningListItem, columns)
+
+        SizeList.__init__(self, None, LIST_GREY, [0, 0], False, parent=parent)
+
+        self.header.SetBackgroundColour(wx.WHITE)
+        self.header.SetBorderColour(SEPARATOR_GREY)
+
+    def GetManager(self):
+        if getattr(self, 'manager', None) == None:
+            self.manager = CreditMiningSearchManager(self)
+        return self.manager
+
+    @warnWxThread
+    def CreateHeader(self, parent):
+        if self.guiutility.frame.top_bg:
+            header = FancyPanel(parent, border=wx.BOTTOM)
+            text = wx.StaticText(header, -1, 'Investment overview')
+
+            def OnAddSource(event):
+                dlg = AddBoostingSource(None)
+                if dlg.ShowModal() == wx.ID_OK:
+                    source, archive = dlg.GetValue()
+                    if source:
+                        self.boosting_manager.add_source(source)
+                        self.boosting_manager.set_archive(source, archive)
+                dlg.Destroy()
+
+            def OnRemoveSource(event):
+                dlg = RemoveBoostingSource(None)
+                if dlg.ShowModal() == wx.ID_OK and dlg.GetValue():
+                    self.boosting_manager.remove_source(dlg.GetValue())
+                    self.GetManager().refresh()
+                dlg.Destroy()
+
+            addsource = LinkStaticText(header, 'Add', icon=None)
+            addsource.Bind(wx.EVT_LEFT_UP, OnAddSource)
+            removesource = LinkStaticText(header, 'Remove', icon=None)
+            removesource.Bind(wx.EVT_LEFT_UP, OnRemoveSource)
+            self.b_up = wx.StaticText(header, -1, 'Total bytes up: -')
+            self.b_down = wx.StaticText(header, -1, 'Total bytes down: -')
+            self.s_up = wx.StaticText(header, -1, 'Total speed up: -')
+            self.s_down = wx.StaticText(header, -1, 'Total speed down: -')
+            self.iv_sum = wx.StaticText(header, -1, 'Investment summary: -')
+            _set_font(text, size_increment=2, fontweight=wx.FONTWEIGHT_BOLD)
+            sizer = wx.BoxSizer(wx.VERTICAL)
+            sizer.AddStretchSpacer()
+            titleSizer = wx.BoxSizer(wx.HORIZONTAL)
+            titleSizer.Add(text, 0, wx.ALIGN_BOTTOM | wx.RIGHT, 5)
+            titleSizer.Add(wx.StaticText(header, -1, '('), 0, wx.ALIGN_BOTTOM)
+            titleSizer.Add(addsource, 0, wx.ALIGN_BOTTOM)
+            titleSizer.Add(wx.StaticText(header, -1, '/'), 0, wx.ALIGN_BOTTOM)
+            titleSizer.Add(removesource, 0, wx.ALIGN_BOTTOM)
+            titleSizer.Add(wx.StaticText(header, -1, ' boosting source)'), 0, wx.ALIGN_BOTTOM)
+            sizer.Add(titleSizer, 0, wx.LEFT | wx.BOTTOM, 5)
+            sizer.Add(self.b_up, 0, wx.LEFT, 5)
+            sizer.Add(self.b_down, 0, wx.LEFT, 5)
+            sizer.Add(self.s_up, 0, wx.LEFT, 5)
+            sizer.Add(self.s_down, 0, wx.LEFT, 5)
+            sizer.Add(self.iv_sum, 0, wx.LEFT, 5)
+            sizer.AddStretchSpacer()
+            header.SetSizer(sizer)
+            header.SetMinSize((-1, 100))
+        else:
+            raise NotYetImplementedException('')
+
+        return header
+
+    @warnWxThread
+    def CreateFooter(self, parent):
+        self.list.ShowMessage("No credit mining data available.")
+        footer = ListFooter(parent, radius=0)
+        footer.SetMinSize((-1, 0))
+        return footer
+
+    @warnWxThread
+    def CreateSource(self, parent, item):
+        torrent = self.boosting_manager.torrents.get(item.original_data.infohash, None)
+        text = torrent.get('source', '')
+        text = text[:30] + '..' if len(text) > 32 else text
+        return wx.StaticText(parent, -1, text)
+
+    def OnExpand(self, item):
+        List.OnExpand(self, item)
+        return True
+
+    @warnWxThread
+    def RefreshItems(self, dslist, magnetlist):
+        didStateChange, _, _ = SizeList.RefreshItems(self, dslist, magnetlist, rawdata=True)
+
+        newFilter = self.newfilter
+
+        new_keys = self.boosting_manager.torrents.keys()
+        old_keys = getattr(self, 'old_keys', [])
+        if len(new_keys) != len(old_keys):
+            self.GetManager().refresh_if_exists(new_keys, force=True)
+            self.old_keys = new_keys
+
+        if didStateChange:
+            if self.statefilter != None:
+                self.list.SetData()  # basically this means execute filter again
+
+        boosting_dslist = [ds for ds in dslist if ds.get_download().get_def().get_infohash() in new_keys]
+        for item in self.list.items.itervalues():
+            ds = item.original_data.ds
+            if ds:
+
+                if not ds in boosting_dslist:
+                    continue
+
+                if ds.get_seeding_statistics():
+                    seeding_stats = ds.get_seeding_statistics()
+                    bytes_up = seeding_stats['total_up']
+                    bytes_down = seeding_stats['total_down']
+
+                    item.RefreshColumn(1, size_format(bytes_up) + ' / ' + size_format(bytes_down))
+                    if bytes_down:
+                        item.RefreshColumn(6, '%f' %(float(bytes_up)/float(bytes_down)))
+
+                #refresh seeder/leecher
+                it_seeder = self.boosting_manager.torrents[item.original_data.infohash]['num_seeders']
+                it_leecher = self.boosting_manager.torrents[item.original_data.infohash]['num_leechers']
+                item.RefreshColumn(2, '%d / %d' %(it_seeder, it_leecher))
+
+                item.SetSelectedColour(wx.Colour(255, 175, 175))
+                item.SetDeselectedColour(wx.Colour(255, 200, 200))
+                item.SetExpandedColour(wx.Colour(255, 150, 150))
+                item.SetExpandedAndSelectedColour(wx.Colour(255, 125, 125))
+
+                speed_up = ds.get_current_speed('up') if ds else 0
+                speed_down = ds.get_current_speed('down') if ds else 0
+
+                item.RefreshColumn(0, speed_format(speed_up) + ' / ' + speed_format(speed_down))
+
+            else:
+                item.SetSelectedColour(LIST_SELECTED)
+                item.SetDeselectedColour(LIST_DESELECTED)
+                item.SetExpandedColour(LIST_EXPANDED)
+                item.SetExpandedAndSelectedColour(LIST_DARKBLUE)
+
+                item.RefreshColumn(0, '- / -')
+
+            if item.original_data.infohash in self.boosting_manager.torrents:
+                is_dup = self.boosting_manager.torrents[item.original_data.infohash].get('is_duplicate', None)
+                item.RefreshColumn(3, ('*' if is_dup else '**') if is_dup != None else '')
+
+
+
+        seeding_stats = [ds.get_seeding_statistics() for ds in boosting_dslist if ds.get_seeding_statistics()]
+        tot_bytes_up = sum([stat['total_up'] for stat in seeding_stats])
+        tot_bytes_dwn = sum([stat['total_down'] for stat in seeding_stats])
+        self.b_up.SetLabel('Total bytes up: ' + size_format(tot_bytes_up))
+        self.b_down.SetLabel('Total bytes down: ' + size_format(tot_bytes_dwn))
+
+        if tot_bytes_dwn:
+            self.iv_sum.SetLabel(' Investment summary: %f' %(float(tot_bytes_up)/float(tot_bytes_dwn)))
+
+        self.s_up.SetLabel('Total speed up: ' + speed_format(sum([ds.get_current_speed('up') for ds in boosting_dslist])))
+        self.s_down.SetLabel('Total speed down: ' + speed_format(sum([ds.get_current_speed('down') for ds in boosting_dslist])))
+
+        if newFilter:
+            self.newfilter = False
+
+        self.oldDS = dict([(infohash, item.original_data.ds) for infohash, item in self.list.items.iteritems()])
+
+    @warnWxThread
+    def SetData(self, data):
+        SizeList.SetData(self, data)
+
+        if len(data) > 0:
+            data = [(file.infohash, ['- / -', '- / -', '%d / %d' % (file.num_seeders, file.num_leechers), '', file.infohash, '', '-1'], file, CreditMiningListItem) for file in data]
+        else:
+            self.list.ShowMessage("No credit mining data available.")
+            self.SetNrResults(0)
+
+        self.list.SetData(data)
+
+    @warnWxThread
+    def RefreshData(self, key, data):
+        List.RefreshData(self, key, data)
+
+        data = (data.infohash, ['-', '-', '%d / %d' % (data.num_seeders, data.num_leechers), '', data.infohash, '','-1'], data)
+        self.list.RefreshData(key, data)
+
+    def SetNrResults(self, nr):
+        highlight = nr > self.nr_results and self.initnumitems
+        SizeList.SetNrResults(self, nr)
+
+        actitem = self.guiutility.frame.actlist.GetItem(5)
+        num_items = getattr(actitem, 'num_items', None)
+        if num_items:
+            num_items.SetValue(str(nr))
+            actitem.hSizer.Layout()
+            if highlight:
+                actitem.Highlight()
+            self.initnumitems = True
+
+    def MatchFilter(self, item):
+        return True
+
+    def MatchFFilter(self, item):
+        return True
+
+
 class ChannelList(List):
 
     def __init__(self, parent):

Tribler/Policies/BoostingManager.py

@@ -251,6 +251,14 @@ def remove_source(self, source_key):
         if source_key in self.boosting_sources:
             source = self.boosting_sources.pop(source_key)
             source.kill_tasks()
+            logger.info("Removed source %s", source_key)
+
+            rm_torrents = [torrent for _, torrent in self.torrents.items() if torrent['source'] == source_key]
+            map(self.stop_download,rm_torrents)
+            logger.info("Torrents download stopped")
+
+            map(lambda x:self.torrents.pop(x["metainfo"].get_infohash(), None), rm_torrents)
+            logger.info("Removing from possible swarms")
 
     def compare_torrents(self, t1, t2):
         # pylint: disable=no-self-use, bad-builtin
@@ -373,7 +381,9 @@ def do_start():
             dscfg.set_safe_seeding(False)
 
             preload = torrent.get('preload', False)
-            logger.info("Starting %s preload %s", hexlify(torrent["metainfo"].get_infohash()), preload)
+            logger.info("Starting %s %d/%d preload %s",
+                        hexlify(torrent["metainfo"].get_infohash()),
+                        torrent["num_seeders"],torrent["num_leechers"], preload)
             torrent['download'] = self.session.lm.add(torrent['metainfo'], dscfg, pstate=torrent.get('pstate', None),
                                                       hidden=True, share_mode=not preload, checkpoint_disabled=True)
             torrent['download'].set_priority(torrent.get('prio', 1))
@@ -405,7 +415,7 @@ def _select_torrent(self):
 
         if self.policy is not None and torrents:
 
-            logger.debug("Selecting from %s torrents", len(torrents))
+            logger.info("Selecting from %s torrents", len(torrents))
 
             # Determine which torrent to start and which to stop.
             torrents_start, torrents_stop = self.policy.apply(
@@ -582,8 +592,8 @@ def showTorrent(torrent):
                 self.torrents[infohash]['creation_date'] = torrent.creation_date
                 self.torrents[infohash]['length'] = torrent.tdef.get_length()
                 self.torrents[infohash]['num_files'] = len(torrent.files)
-                self.torrents[infohash]['num_seeders'] = torrent.swarminfo[0]
-                self.torrents[infohash]['num_leechers'] = torrent.swarminfo[1]
+                self.torrents[infohash]['num_seeders'] = torrent.swarminfo[0] or -1
+                self.torrents[infohash]['num_leechers'] = torrent.swarminfo[1] or -1
 
                 del self.unavail_torrent[infohash]
 
@@ -599,7 +609,9 @@ def showTorrent(torrent):
             for k,t in self.unavail_torrent.items():
                 startWorker(doGui, self.gui_util.torrentsearch_manager.loadTorrent,
                             wargs=(t,), wkwargs={'callback': showTorrent})
-            self.session.lm.threadpool.add_task(self._check_tor, 100, task_name=str(self.source)+"_checktor")
+
+            if not self.session.lm.threadpool.is_pending_task_active(str(self.source)+"_checktor"):
+                self.session.lm.threadpool.add_task(self._check_tor, 100, task_name=str(self.source)+"_checktor")
 
     def _update(self):
         if len(self.torrents) < self.max_torrents:
@@ -693,6 +705,12 @@ def _update(self):
 
             self.session.lm.threadpool.add_task(self._update, self.interval, task_name=str(self.source)+"_update")
 
+    def kill_tasks(self):
+        BoostingSource.kill_tasks(self)
+
+        #stop updating
+        self.session.lm.threadpool.cancel_pending_task(str(self.source)+"_update")
+
 
 class DirectorySource(BoostingSource):