diff --git a/Tribler/Main/tribler_main.py b/Tribler/Main/tribler_main.py index cec58b1b7c6..c97dee73814 100755 --- a/Tribler/Main/tribler_main.py +++ b/Tribler/Main/tribler_main.py @@ -865,6 +865,15 @@ def OnExit(self): self.webUI.stop() self.webUI.delInstance() + if self.boosting_manager: + + #remove credit mining data + #TODO(ardhi) : not persistent mode only + import shutil; shutil.rmtree(self.boosting_manager.credit_mining_path) + + self.boosting_manager.shutdown() + self.boosting_manager.del_instance() + if self.frame: self.frame.Destroy() self.frame = None diff --git a/Tribler/Main/vwxGUI/list.py b/Tribler/Main/vwxGUI/list.py index d1f0e70fd08..fb3c46974aa 100644 --- a/Tribler/Main/vwxGUI/list.py +++ b/Tribler/Main/vwxGUI/list.py @@ -302,6 +302,70 @@ def downloadStarted(self, _): self.refresh() +class CreditMiningSearchManager(BaseManager): + + def __init__(self, list): + BaseManager.__init__(self, list) + self.boosting_manager = BoostingManager.get_instance() + self.library_manager = self.guiutility.library_manager + + def refresh(self): + startWorker(self._on_data, self.getHitsInCategory, uId=u"CreditMiningSearchManager_refresh", retryOnBusy=True, priority=GUI_PRI_DISPERSY) + + def getTorrentFromInfohash(self, infohash): + torrent = self.boosting_manager.torrents.get(infohash, None) + if torrent: + t = LibraryTorrent('', infohash, name=torrent['name'], length=torrent['length'], category='', status='', + num_seeders=torrent['num_seeders'], num_leechers=torrent['num_leechers']) + t.torrent_db = self.library_manager.torrent_db + t.channelcast_db = self.library_manager.channelcast_db + # t.channel + self.library_manager.addDownloadState(t) + return t + + def getHitsInCategory(self): + hits = [self.getTorrentFromInfohash(infohash) for infohash in self.boosting_manager.torrents.keys()] + return [len(hits), hits] + + def refresh_partial(self, ids): + for infohash in ids: + startWorker(self.list.RefreshDelayedData, self.getTorrentFromInfohash, cargs=(infohash,), wargs=(infohash,), retryOnBusy=True, priority=GUI_PRI_DISPERSY) + + def refresh_if_exists(self, infohashes, force=False): + if any([self.boosting_manager.torrents.has_key(infohash) for infohash in infohashes]): + print >> sys.stderr, long(time()), "Scheduling a refresh, missing some infohashes in the Credit Mining overview" + self.refresh() + else: + print >> sys.stderr, long(time()), "Not scheduling a refresh" + + def refresh_or_expand(self, infohash): + if not self.list.InList(infohash): + def select(delayedResult): + delayedResult.get() + self.refresh_or_expand(infohash) + + startWorker(select, self.refresh_partial, wargs=([infohash],), priority=GUI_PRI_DISPERSY) + else: + self.list.Select(infohash) + + @forceWxThread + def _on_data(self, delayedResult): + total_items, data = delayedResult.get() + self.list.SetData(data) + self.list.Layout() + + def torrentUpdated(self, infohash): + if self.list.InList(infohash): + self.do_or_schedule_partial([infohash]) + + def torrentsUpdated(self, infohashes): + infohashes = [infohash for infohash in infohashes if self.list.InList(infohash)] + self.do_or_schedule_partial(infohashes) + + def downloadStarted(self, infohash): + self.refresh() + + class ChannelSearchManager(BaseManager): def __init__(self, list): @@ -1945,6 +2009,232 @@ def GetFilterMessage(self, empty=False): return header, message +class CreditMiningList(SizeList): + + def __init__(self, parent): + self.boosting_manager = BoostingManager.get_instance() + self.guiutility = GUIUtility.getInstance() + self.utility = self.guiutility.utility + + self.statefilter = None + self.newfilter = False + self.prevStates = {} + self.oldDS = {} + + self.initnumitems = False + + columns = [{'name': 'Speed up/down', 'width': '32em', 'autoRefresh': False}, + {'name': 'Bytes up/down', 'width': '32em', 'autoRefresh': False}, + {'name': 'Seeders/leechers', 'width': '27em', 'autoRefresh': False}, + {'name': 'Duplicate', 'showColumname': False, 'width': '2em'}, + {'name': 'Hash', 'width': '27em', 'fmt': lambda ih: ih.encode('hex')[:10]}, + {'name': 'Source', 'width': '40em', 'type': 'method', 'method': self.CreateSource}, + {'name': 'Investment status', 'width': '32em', 'autoRefresh': False}] + + columns = self.guiutility.SetColumnInfo(CreditMiningListItem, columns) + ColumnsManager.getInstance().setColumns(CreditMiningListItem, columns) + + SizeList.__init__(self, None, LIST_GREY, [0, 0], False, parent=parent) + + self.header.SetBackgroundColour(wx.WHITE) + self.header.SetBorderColour(SEPARATOR_GREY) + + def GetManager(self): + if getattr(self, 'manager', None) == None: + self.manager = CreditMiningSearchManager(self) + return self.manager + + @warnWxThread + def CreateHeader(self, parent): + if self.guiutility.frame.top_bg: + header = FancyPanel(parent, border=wx.BOTTOM) + text = wx.StaticText(header, -1, 'Investment overview') + + def OnAddSource(event): + dlg = AddBoostingSource(None) + if dlg.ShowModal() == wx.ID_OK: + source, archive = dlg.GetValue() + if source: + self.boosting_manager.add_source(source) + self.boosting_manager.set_archive(source, archive) + dlg.Destroy() + + def OnRemoveSource(event): + dlg = RemoveBoostingSource(None) + if dlg.ShowModal() == wx.ID_OK and dlg.GetValue(): + self.boosting_manager.remove_source(dlg.GetValue()) + self.GetManager().refresh() + dlg.Destroy() + + addsource = LinkStaticText(header, 'Add', icon=None) + addsource.Bind(wx.EVT_LEFT_UP, OnAddSource) + removesource = LinkStaticText(header, 'Remove', icon=None) + removesource.Bind(wx.EVT_LEFT_UP, OnRemoveSource) + self.b_up = wx.StaticText(header, -1, 'Total bytes up: -') + self.b_down = wx.StaticText(header, -1, 'Total bytes down: -') + self.s_up = wx.StaticText(header, -1, 'Total speed up: -') + self.s_down = wx.StaticText(header, -1, 'Total speed down: -') + self.iv_sum = wx.StaticText(header, -1, 'Investment summary: -') + _set_font(text, size_increment=2, fontweight=wx.FONTWEIGHT_BOLD) + sizer = wx.BoxSizer(wx.VERTICAL) + sizer.AddStretchSpacer() + titleSizer = wx.BoxSizer(wx.HORIZONTAL) + titleSizer.Add(text, 0, wx.ALIGN_BOTTOM | wx.RIGHT, 5) + titleSizer.Add(wx.StaticText(header, -1, '('), 0, wx.ALIGN_BOTTOM) + titleSizer.Add(addsource, 0, wx.ALIGN_BOTTOM) + titleSizer.Add(wx.StaticText(header, -1, '/'), 0, wx.ALIGN_BOTTOM) + titleSizer.Add(removesource, 0, wx.ALIGN_BOTTOM) + titleSizer.Add(wx.StaticText(header, -1, ' boosting source)'), 0, wx.ALIGN_BOTTOM) + sizer.Add(titleSizer, 0, wx.LEFT | wx.BOTTOM, 5) + sizer.Add(self.b_up, 0, wx.LEFT, 5) + sizer.Add(self.b_down, 0, wx.LEFT, 5) + sizer.Add(self.s_up, 0, wx.LEFT, 5) + sizer.Add(self.s_down, 0, wx.LEFT, 5) + sizer.Add(self.iv_sum, 0, wx.LEFT, 5) + sizer.AddStretchSpacer() + header.SetSizer(sizer) + header.SetMinSize((-1, 100)) + else: + raise NotYetImplementedException('') + + return header + + @warnWxThread + def CreateFooter(self, parent): + self.list.ShowMessage("No credit mining data available.") + footer = ListFooter(parent, radius=0) + footer.SetMinSize((-1, 0)) + return footer + + @warnWxThread + def CreateSource(self, parent, item): + torrent = self.boosting_manager.torrents.get(item.original_data.infohash, None) + text = torrent.get('source', '') + text = text[:30] + '..' if len(text) > 32 else text + return wx.StaticText(parent, -1, text) + + def OnExpand(self, item): + List.OnExpand(self, item) + return True + + @warnWxThread + def RefreshItems(self, dslist, magnetlist): + didStateChange, _, _ = SizeList.RefreshItems(self, dslist, magnetlist, rawdata=True) + + newFilter = self.newfilter + + new_keys = self.boosting_manager.torrents.keys() + old_keys = getattr(self, 'old_keys', []) + if len(new_keys) != len(old_keys): + self.GetManager().refresh_if_exists(new_keys, force=True) + self.old_keys = new_keys + + if didStateChange: + if self.statefilter != None: + self.list.SetData() # basically this means execute filter again + + boosting_dslist = [ds for ds in dslist if ds.get_download().get_def().get_infohash() in new_keys] + for item in self.list.items.itervalues(): + ds = item.original_data.ds + if ds: + + if not ds in boosting_dslist: + continue + + if ds.get_seeding_statistics(): + seeding_stats = ds.get_seeding_statistics() + bytes_up = seeding_stats['total_up'] + bytes_down = seeding_stats['total_down'] + + item.RefreshColumn(1, size_format(bytes_up) + ' / ' + size_format(bytes_down)) + if bytes_down: + item.RefreshColumn(6, '%f' %(float(bytes_up)/float(bytes_down))) + + #refresh seeder/leecher + it_seeder = self.boosting_manager.torrents[item.original_data.infohash]['num_seeders'] + it_leecher = self.boosting_manager.torrents[item.original_data.infohash]['num_leechers'] + item.RefreshColumn(2, '%d / %d' %(it_seeder, it_leecher)) + + item.SetSelectedColour(wx.Colour(255, 175, 175)) + item.SetDeselectedColour(wx.Colour(255, 200, 200)) + item.SetExpandedColour(wx.Colour(255, 150, 150)) + item.SetExpandedAndSelectedColour(wx.Colour(255, 125, 125)) + + speed_up = ds.get_current_speed('up') if ds else 0 + speed_down = ds.get_current_speed('down') if ds else 0 + + item.RefreshColumn(0, speed_format(speed_up) + ' / ' + speed_format(speed_down)) + + else: + item.SetSelectedColour(LIST_SELECTED) + item.SetDeselectedColour(LIST_DESELECTED) + item.SetExpandedColour(LIST_EXPANDED) + item.SetExpandedAndSelectedColour(LIST_DARKBLUE) + + item.RefreshColumn(0, '- / -') + + if item.original_data.infohash in self.boosting_manager.torrents: + is_dup = self.boosting_manager.torrents[item.original_data.infohash].get('is_duplicate', None) + item.RefreshColumn(3, ('*' if is_dup else '**') if is_dup != None else '') + + + + seeding_stats = [ds.get_seeding_statistics() for ds in boosting_dslist if ds.get_seeding_statistics()] + tot_bytes_up = sum([stat['total_up'] for stat in seeding_stats]) + tot_bytes_dwn = sum([stat['total_down'] for stat in seeding_stats]) + self.b_up.SetLabel('Total bytes up: ' + size_format(tot_bytes_up)) + self.b_down.SetLabel('Total bytes down: ' + size_format(tot_bytes_dwn)) + + if tot_bytes_dwn: + self.iv_sum.SetLabel(' Investment summary: %f' %(float(tot_bytes_up)/float(tot_bytes_dwn))) + + self.s_up.SetLabel('Total speed up: ' + speed_format(sum([ds.get_current_speed('up') for ds in boosting_dslist]))) + self.s_down.SetLabel('Total speed down: ' + speed_format(sum([ds.get_current_speed('down') for ds in boosting_dslist]))) + + if newFilter: + self.newfilter = False + + self.oldDS = dict([(infohash, item.original_data.ds) for infohash, item in self.list.items.iteritems()]) + + @warnWxThread + def SetData(self, data): + SizeList.SetData(self, data) + + if len(data) > 0: + data = [(file.infohash, ['- / -', '- / -', '%d / %d' % (file.num_seeders, file.num_leechers), '', file.infohash, '', '-1'], file, CreditMiningListItem) for file in data] + else: + self.list.ShowMessage("No credit mining data available.") + self.SetNrResults(0) + + self.list.SetData(data) + + @warnWxThread + def RefreshData(self, key, data): + List.RefreshData(self, key, data) + + data = (data.infohash, ['-', '-', '%d / %d' % (data.num_seeders, data.num_leechers), '', data.infohash, '','-1'], data) + self.list.RefreshData(key, data) + + def SetNrResults(self, nr): + highlight = nr > self.nr_results and self.initnumitems + SizeList.SetNrResults(self, nr) + + actitem = self.guiutility.frame.actlist.GetItem(5) + num_items = getattr(actitem, 'num_items', None) + if num_items: + num_items.SetValue(str(nr)) + actitem.hSizer.Layout() + if highlight: + actitem.Highlight() + self.initnumitems = True + + def MatchFilter(self, item): + return True + + def MatchFFilter(self, item): + return True + + class ChannelList(List): def __init__(self, parent): diff --git a/Tribler/Policies/BoostingManager.py b/Tribler/Policies/BoostingManager.py index 98bda736c25..ffa3ccfabb7 100644 --- a/Tribler/Policies/BoostingManager.py +++ b/Tribler/Policies/BoostingManager.py @@ -251,6 +251,14 @@ def remove_source(self, source_key): if source_key in self.boosting_sources: source = self.boosting_sources.pop(source_key) source.kill_tasks() + logger.info("Removed source %s", source_key) + + rm_torrents = [torrent for _, torrent in self.torrents.items() if torrent['source'] == source_key] + map(self.stop_download,rm_torrents) + logger.info("Torrents download stopped") + + map(lambda x:self.torrents.pop(x["metainfo"].get_infohash(), None), rm_torrents) + logger.info("Removing from possible swarms") def compare_torrents(self, t1, t2): # pylint: disable=no-self-use, bad-builtin @@ -373,7 +381,9 @@ def do_start(): dscfg.set_safe_seeding(False) preload = torrent.get('preload', False) - logger.info("Starting %s preload %s", hexlify(torrent["metainfo"].get_infohash()), preload) + logger.info("Starting %s %d/%d preload %s", + hexlify(torrent["metainfo"].get_infohash()), + torrent["num_seeders"],torrent["num_leechers"], preload) torrent['download'] = self.session.lm.add(torrent['metainfo'], dscfg, pstate=torrent.get('pstate', None), hidden=True, share_mode=not preload, checkpoint_disabled=True) torrent['download'].set_priority(torrent.get('prio', 1)) @@ -405,7 +415,7 @@ def _select_torrent(self): if self.policy is not None and torrents: - logger.debug("Selecting from %s torrents", len(torrents)) + logger.info("Selecting from %s torrents", len(torrents)) # Determine which torrent to start and which to stop. torrents_start, torrents_stop = self.policy.apply( @@ -582,8 +592,8 @@ def showTorrent(torrent): self.torrents[infohash]['creation_date'] = torrent.creation_date self.torrents[infohash]['length'] = torrent.tdef.get_length() self.torrents[infohash]['num_files'] = len(torrent.files) - self.torrents[infohash]['num_seeders'] = torrent.swarminfo[0] - self.torrents[infohash]['num_leechers'] = torrent.swarminfo[1] + self.torrents[infohash]['num_seeders'] = torrent.swarminfo[0] or -1 + self.torrents[infohash]['num_leechers'] = torrent.swarminfo[1] or -1 del self.unavail_torrent[infohash] @@ -599,7 +609,9 @@ def showTorrent(torrent): for k,t in self.unavail_torrent.items(): startWorker(doGui, self.gui_util.torrentsearch_manager.loadTorrent, wargs=(t,), wkwargs={'callback': showTorrent}) - self.session.lm.threadpool.add_task(self._check_tor, 100, task_name=str(self.source)+"_checktor") + + if not self.session.lm.threadpool.is_pending_task_active(str(self.source)+"_checktor"): + self.session.lm.threadpool.add_task(self._check_tor, 100, task_name=str(self.source)+"_checktor") def _update(self): if len(self.torrents) < self.max_torrents: @@ -693,6 +705,12 @@ def _update(self): self.session.lm.threadpool.add_task(self._update, self.interval, task_name=str(self.source)+"_update") + def kill_tasks(self): + BoostingSource.kill_tasks(self) + + #stop updating + self.session.lm.threadpool.cancel_pending_task(str(self.source)+"_update") + class DirectorySource(BoostingSource): diff --git a/logger.conf b/logger.conf index be29b4b016f..87888b5f344 100644 --- a/logger.conf +++ b/logger.conf @@ -1,5 +1,5 @@ [loggers] -keys=root,candidates,twisted,MetadataInjector,HiddenTunnelCommunity,TunnelMain,TunnelLogger,BarterCommunity,BarterCommunityCrawler +keys=root,candidates,twisted,MetadataInjector,HiddenTunnelCommunity,TunnelMain,TunnelLogger,BarterCommunity,BarterCommunityCrawler,Circuit,Torrent,BaseManager [handlers] keys=debugging,default @@ -17,6 +17,18 @@ qualname=MetadataInjector handlers=default propagate=0 +[logger_Torrent] +level=INFO +qualname=Torrent +handlers=default +propagate=0 + +[logger_BaseManager] +level=INFO +qualname=BaseManager +handlers=default +propagate=0 + [logger_BarterCommunity] level=ERROR qualname=BarterCommunity @@ -36,11 +48,17 @@ handlers=default propagate=0 [logger_TunnelLogger] -level=ERROR +level=CRITICAL qualname=TunnelLogger handlers=default propagate=0 +[logger_Circuit] +level=CRITICAL +qualname=Circuit +handlers=default +propagate=0 + [logger_TunnelMain] level=INFO qualname=TunnelMain