Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

GuiDBTuples credit mining removal #2398

Merged
merged 4 commits into from
Oct 19, 2016
Merged

GuiDBTuples credit mining removal #2398

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7a1970e
Remove GUIDBTuples from Credit mining
ardhipoetra Jul 2, 2016
1e20d37
Refactor GUI to cope with GUIDBTuples removal
ardhipoetra Jul 2, 2016
7c7c99b
Removed session and task check
ardhipoetra Jul 19, 2016
5b6a6f0
Add test for native loading
ardhipoetra Sep 26, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions Tribler/Main/vwxGUI/CreditMiningPanel.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -382,9 +382,9 @@ def show_source_info(self, data):

self.source_label.SetLabel("Source : Channel (stored)")
self.source_name.SetLabel("Name : " + data.get_source_text())
self.torrent_num.SetLabel("# Torrents : " + str(data.channel.nr_torrents))
self.last_updt.SetLabel("Latest update : " + format_time(data.channel.modified))
self.votes_num.SetLabel('Favorite votes : ' + str(data.channel.nr_favorites))
self.torrent_num.SetLabel("# Torrents : " + str(data.channel_dict[4]))
self.last_updt.SetLabel("Latest update : " + format_time(data.channel_dict[8]))
self.votes_num.SetLabel('Favorite votes : ' + str(data.channel_dict[5]))
self.status_cm.SetLabel("Active" if data.enabled else "Inactive")

debug_str = hexlify(data.source)
Expand Down
97 changes: 51 additions & 46 deletions Tribler/Policies/BoostingSource.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,12 @@
import os
import re
import urllib
from binascii import hexlify
from binascii import hexlify, unhexlify
from hashlib import sha1
import feedparser

import feedparser
import libtorrent as lt

from twisted.internet import defer
from twisted.internet import reactor
from twisted.internet.defer import CancelledError
Expand All @@ -24,8 +25,7 @@
from Tribler.Core.TorrentDef import TorrentDef
from Tribler.Core.simpledefs import NTFY_INSERT, NTFY_TORRENTS, NTFY_UPDATE
from Tribler.Core.version import version_id
from Tribler.Main.Utility.GuiDBTuples import Torrent, Channel
from Tribler.Policies.credit_mining_util import TorrentManagerCM, ent2chr
from Tribler.Policies.credit_mining_util import ent2chr
from Tribler.community.allchannel.community import AllChannelCommunity
from Tribler.community.channel.community import ChannelCommunity
from Tribler.dispersy.exception import CommunityNotFoundException
Expand Down Expand Up @@ -59,8 +59,6 @@ def __init__(self, session, source, boost_settings, torrent_insert_cb):
self.min_connection = boost_settings.min_connection_start
self.min_channels = boost_settings.min_channels_start

self.torrent_mgr = TorrentManagerCM(session)

self._logger = logging.getLogger(BoostingSource.__name__)

self.boosting_manager = self.session.lm.boosting_manager
Expand Down Expand Up @@ -129,17 +127,6 @@ def get_source_text(self):
def _on_err(self, err_msg):
self._logger.error(err_msg)

def check_and_register_task(self, name, task, delay=None, value=None, interval=None):
"""
Helper function to avoid assertion in register task.

It will register task if it has not already registered
"""
task_ret = None
if not self.is_pending_task_active(name):
task_ret = self.register_task(name, task, delay, value, interval)

return task_ret

class ChannelSource(BoostingSource):
"""
Expand All @@ -150,16 +137,18 @@ def __init__(self, session, dispersy_cid, boost_settings, torrent_insert_cb):

self.channel_id = None

self.channel = None
self.channel_dict = None
self.community = None
self.database_updated = True

self.check_torrent_interval = 10
self.dispersy_cid = dispersy_cid

self.torrent_db = self.session.open_dbhandler(NTFY_TORRENTS)
self.session.add_observer(self._on_database_updated, NTFY_TORRENTS, [NTFY_INSERT, NTFY_UPDATE])

self.unavail_torrent = {}
self.loaded_torrent = {}

def kill_tasks(self):
BoostingSource.kill_tasks(self)
Expand Down Expand Up @@ -200,11 +189,9 @@ def get_channel_id():
if self.community and self.community._channel_id:
self.channel_id = self.community._channel_id

channel_dict = self.channelcast_db.getChannel(self.channel_id)
self.channel = Channel(*channel_dict)

task_call = self.check_and_register_task(str(self.source) + "_update",
LoopingCall(self._update)).start(self.interval, now=True)
self.channel_dict = self.channelcast_db.getChannel(self.channel_id)
task_call = self.register_task(str(self.source) + "_update",
LoopingCall(self._update)).start(self.interval, now=True)
if task_call:
self._logger.debug("Registering update call")

Expand All @@ -225,22 +212,23 @@ def showtorrent(torrent):
"""
assembly torrent data, call the callback
"""
if torrent.files:
infohash = torrent.infohash
if torrent.get_files() and infohash in self.unavail_torrent:
if len(self.torrents) >= self.max_torrents:
self._logger.debug("Max torrents in source reached. Not adding %s", torrent.infohash)
del self.unavail_torrent[torrent.infohash]
return

infohash = torrent.infohash
self._logger.debug("[ChannelSource] Got torrent %s", hexlify(infohash))
self.torrents[infohash] = {}
self.torrents[infohash]['name'] = torrent.name
self.torrents[infohash]['metainfo'] = torrent.tdef
self.torrents[infohash]['creation_date'] = torrent.creation_date
self.torrents[infohash]['length'] = torrent.tdef.get_length()
self.torrents[infohash]['num_files'] = len(torrent.files)
self.torrents[infohash]['num_seeders'] = torrent.swarminfo[0] or 0
self.torrents[infohash]['num_leechers'] = torrent.swarminfo[1] or 0
self.torrents[infohash]['name'] = torrent.get_name()
self.torrents[infohash]['metainfo'] = torrent
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does the torrent object really represent the metainfo of the torrent?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think yes, torrent object is a TorrentDef type

self.torrents[infohash]['creation_date'] = torrent.get_creation_date()
self.torrents[infohash]['length'] = torrent.get_length()
self.torrents[infohash]['num_files'] = len(torrent.get_files())
#TODO(ardhi) get seeder/leecher from db
self.torrents[infohash]['num_seeders'] = 0
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does these default values (0) have any consequences for the picking of the torrents in the credit mining code (the boosting policies)?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It a default/start value. It will be replaced later when scraping the torrent. It will be better if I can provide number of seeder/leecher right away from database, but I can't find any way to do it now.

self.torrents[infohash]['num_leechers'] = 0
self.torrents[infohash]['enabled'] = self.enabled

# seeding stats from DownloadState
Expand All @@ -252,11 +240,10 @@ def showtorrent(torrent):
self.torrent_insert_callback(self.source, infohash, self.torrents[infohash])
self.database_updated = False

self._logger.debug("Unavailable #torrents : %d from %s", len(self.unavail_torrent), hexlify(self.source))

if len(self.unavail_torrent) and self.enabled:
self._logger.debug("Unavailable #torrents : %d from %s", len(self.unavail_torrent), hexlify(self.source))
for torrent in self.unavail_torrent.values():
self.torrent_mgr.load_torrent(torrent, showtorrent)
self._load_torrent(torrent[2]).addCallback(showtorrent)

def _update(self):
if len(self.torrents) < self.max_torrents and self.database_updated:
Expand All @@ -268,17 +255,13 @@ def _update(self):
torrent_values = self.channelcast_db.getTorrentsFromChannelId(self.channel_id, True, CHANTOR_DB,
self.max_torrents)

listtor = self.torrent_mgr.create_torrents(torrent_values, True,
{self.channel_id: self.channelcast_db.getChannel(
self.channel_id)})

# dict {key_infohash(binary):Torrent(object-GUIDBTuple)}
self.unavail_torrent.update({t.infohash: t for t in listtor if t.infohash not in self.torrents})
# dict {key_infohash(binary):Torrent(tuples)}
self.unavail_torrent.update({t[2]: t for t in torrent_values if t[2] not in self.torrents})

# it's highly probable the checktor function is running at this time (if it's already running)
# if not running, start the checker

task_call = self.check_and_register_task(hexlify(self.source) + "_checktor", LoopingCall(self._check_tor))
task_call = self.register_task(hexlify(self.source) + "_checktor", LoopingCall(self._check_tor))
if task_call:
self._logger.debug("Registering check torrent function")
task_call.start(self.check_torrent_interval, now=True)
Expand All @@ -287,7 +270,32 @@ def _on_database_updated(self, dummy_subject, dummy_change_type, dummy_infohash)
self.database_updated = True

def get_source_text(self):
return self.channel.name if self.channel else None
return str(self.channel_dict[2]) if self.channel_dict else None

def _load_torrent(self, infohash):
"""
function to download a torrent by infohash and call a callback afterwards
with TorrentDef object as parameter.
"""

def add_to_loaded(infohash_str):
"""
function to add loaded infohash to memory
"""
self.loaded_torrent[unhexlify(infohash_str)].callback(
TorrentDef.load_from_memory(self.session.get_collected_torrent(unhexlify(infohash_str))))

if infohash not in self.loaded_torrent:
self.loaded_torrent[infohash] = defer.Deferred()

if not self.session.has_collected_torrent(infohash):
if self.session.has_download(infohash):
return
self.session.download_torrentfile(infohash, add_to_loaded, 0)

deferred_load = self.loaded_torrent[infohash]

return deferred_load


class RSSFeedSource(BoostingSource):
Expand Down Expand Up @@ -386,9 +394,6 @@ def __cb_body(body_bin, item_torrent_entry):
self.torrent_db.addOrGetTorrentID(real_infohash)
self.torrent_db.addExternalTorrent(tdef)

# create Torrent object and store it
self.torrent_mgr.load_torrent(Torrent.fromTorrentDef(tdef))

# Notify the BoostingManager and provide the real infohash.
if self.torrent_insert_callback:
self.torrent_insert_callback(self.source, real_infohash, self.torrents[real_infohash])
Expand Down
120 changes: 0 additions & 120 deletions Tribler/Policies/credit_mining_util.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,10 @@
File containing function used in credit mining module.
"""


import os
from binascii import hexlify, unhexlify

from Tribler.Core.TorrentDef import TorrentDef
from Tribler.Core.simpledefs import NTFY_CHANNELCAST
from Tribler.Core.simpledefs import NTFY_TORRENTS
from Tribler.Core.simpledefs import NTFY_VOTECAST
from Tribler.Main.Utility.GuiDBTuples import CollectedTorrent, RemoteTorrent, NotCollectedTorrent, Channel, \
ChannelTorrent
from Tribler.Policies.defs import SIMILARITY_TRESHOLD
from Tribler.dispersy.taskmanager import TaskManager


def validate_source_string(source):
Expand Down Expand Up @@ -85,115 +77,3 @@ def ent2chr(input_str):
code = input_str.group(1)
code_int = int(code) if code.isdigit() else int(code[1:], 16)
return chr(code_int) if code_int < 256 else '?'

# TODO(ardhi) : temporary function until GUI and core code are separated
class TorrentManagerCM(TaskManager):
"""
*Temporary* class to handle load torrent.

Adapted from TorrentManager in SearchGridManager
"""
def __init__(self, session):
super(TorrentManagerCM, self).__init__()

self.session = session
self.torrent_db = self.session.open_dbhandler(NTFY_TORRENTS)
self.channelcast_db = self.session.open_dbhandler(NTFY_CHANNELCAST)
self.votecastdb = self.session.open_dbhandler(NTFY_VOTECAST)

self.dslist = []

def load_torrent(self, torrent, callback=None):
"""
function to load torrent dictionary to torrent object.

From TorrentManager.loadTorrent in SearchGridManager
"""

# session is quitting
if not (self.session and self.session.get_torrent_store() and self.session.lm.torrent_store):
return

if not isinstance(torrent, CollectedTorrent):
if torrent.torrent_id <= 0:
torrent_id = self.torrent_db.getTorrentID(torrent.infohash)
if torrent_id:
torrent.update_torrent_id(torrent_id)

if not self.session.has_collected_torrent(torrent.infohash):
files = []
trackers = []

# see if we have most info in our tables
if isinstance(torrent, RemoteTorrent):
torrent_id = self.torrent_db.getTorrentID(torrent.infohash)
else:
torrent_id = torrent.torrent_id

trackers.extend(self.torrent_db.getTrackerListByTorrentID(torrent_id))

if 'DHT' in trackers:
trackers.remove('DHT')
if 'no-DHT' in trackers:
trackers.remove('no-DHT')

# replacement # self.downloadTorrentfileFromPeers(torrent, None)
if self.session.has_download(torrent.infohash):
return False

if torrent.query_candidates is None or len(torrent.query_candidates) == 0:
self.session.download_torrentfile(torrent.infohash, None, 0)
else:
for candidate in torrent.query_candidates:
self.session.download_torrentfile_from_peer(candidate, torrent.infohash, None, 0)

torrent = NotCollectedTorrent(torrent, files, trackers)

else:
tdef = TorrentDef.load_from_memory(self.session.get_collected_torrent(torrent.infohash))

if torrent.torrent_id <= 0:
del torrent.torrent_id

torrent = CollectedTorrent(torrent, tdef)

# replacement # self.library_manager.addDownloadState(torrent)
for dl_state in self.dslist:
torrent.addDs(dl_state)

# return
if callback is not None:
callback(torrent)
else:
return torrent

def create_torrents(self, tor_values, _, channel_dict):
"""
function to create torrents from channel. Adapted from
ChannelManager in SearchGridManager
"""

#adding new channel from the one that can't be detected from torrent values
fetch_channels = set(hit[0] for hit in tor_values if hit[0] not in channel_dict)
if len(fetch_channels) > 0:
channels_new_dict = self.channelcast_db.getChannels(fetch_channels)
channels = []
for hit in channels_new_dict:
channel = Channel(*hit)
channels.append(channel)

for channel in channels:
channel_dict[channel.id] = channel

# creating torrents
torrents = []
for hit in tor_values:
if hit:
chan_torrent = ChannelTorrent(*hit[1:] + [channel_dict.get(hit[0], None), None])
chan_torrent.torrent_db = self.torrent_db
chan_torrent.channelcast_db = self.channelcast_db

if chan_torrent.name:
torrents.append(chan_torrent)

return torrents
Loading