Merge pull request #100 from Tribler/release-6.1

Release 6.1

Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py

@@ -82,6 +82,7 @@ def __init__(self, session, tdef):
         self.finished_time = 0.0
         self.done = False
         self.pause_after_next_hashcheck = False
+        self.checkpoint_after_next_hashcheck = False
         self.prebuffsize = 5*1024*1024
         self.queue_position = -1
 
@@ -151,11 +152,13 @@ def network_create_engine_wrapper(self, lm_network_engine_wrapper_created_callba
 
         atp = {}
         atp["save_path"] = str(self.dlconfig['saveas'])
-        atp["storage_mode"] = lt.storage_mode_t.storage_mode_sparse
+        # Using full allocation seems to fix issues with streaming certain files.
+        atp["storage_mode"] = lt.storage_mode_t.storage_mode_allocate if self.get_mode() == DLMODE_VOD else lt.storage_mode_t.storage_mode_sparse
         atp["paused"] = True
         atp["auto_managed"] = False
         atp["duplicate_is_error"] = True
 
+        resume_data = pstate.get('engineresumedata', None) if pstate else None
         if not isinstance(self.tdef, TorrentDefNoMetainfo):
             metainfo = self.tdef.get_metainfo()
             torrentinfo = lt.torrent_info(metainfo)
@@ -173,11 +176,15 @@ def network_create_engine_wrapper(self, lm_network_engine_wrapper_created_callba
             self.orig_files = [torrent_file.path for torrent_file in torrentinfo.files()]
 
             atp["ti"] = torrentinfo
-            if pstate and pstate.get('engineresumedata', None):
-                atp["resume_data"] = lt.bencode(pstate['engineresumedata'])
-            print >> sys.stderr, self.tdef.get_name_as_unicode(), pstate.get('engineresumedata', None) if pstate else None
+            if resume_data:
+                atp["resume_data"] = lt.bencode(resume_data)
+            print >> sys.stderr, self.tdef.get_name_as_unicode(), resume_data
         else:
-            atp["info_hash"] = lt.big_number(self.tdef.get_infohash())
+            if self.tdef.get_url():
+                # We prefer to use an url, since it may contain trackers
+                atp["url"] = self.tdef.get_url()
+            else:
+                atp["info_hash"] = lt.big_number(self.tdef.get_infohash())
             atp["name"] = str(self.tdef.get_name())
 
         self.handle = self.ltmgr.add_torrent(self, atp)
@@ -187,8 +194,16 @@ def network_create_engine_wrapper(self, lm_network_engine_wrapper_created_callba
             self.set_selected_files()
             if self.get_mode() == DLMODE_VOD:
                 self.set_vod_mode()
-            if initialdlstatus != DLSTATUS_STOPPED:
+
+            # If we lost resume_data always resume download in order to force checking
+            if initialdlstatus != DLSTATUS_STOPPED or not resume_data: 
                 self.handle.resume()
+
+                # If we only needed to perform checking, pause download after it is complete
+                self.pause_after_next_hashcheck = initialdlstatus == DLSTATUS_STOPPED
+
+        else:
+            print >> sys.stderr, "Could not add torrent to LibtorrentManager", self.tdef.get_name_as_unicode()
 
         with self.dllock:
             self.cew_scheduled = False
@@ -301,48 +316,80 @@ def process_alert(self, alert, alert_type):
 
         if self.handle and self.handle.is_valid():
 
-            status = self.handle.status()
-
             with self.dllock:
 
                 if alert_type == 'metadata_received_alert':
-                    self.metadata = {'info': lt.bdecode(self.handle.get_torrent_info().metadata())}
-                    self.tdef = TorrentDef.load_from_dict(self.metadata)
-                    self.orig_files = [torrent_file.path for torrent_file in lt.torrent_info(self.metadata).files()]
-                    self.set_files()
-
-                    if self.session.lm.rtorrent_handler:
-                        self.session.lm.rtorrent_handler.save_torrent(self.tdef)
-                    elif self.session.lm.torrent_db:
-                        self.session.lm.torrent_db.addExternalTorrent(self.tdef, source = '', extra_info = {'status':'good'}, commit = True)
-
-                    # Checkpoint
-                    (infohash, pstate) = self.network_checkpoint()
-                    checkpoint = lambda : self.session.lm.save_download_pstate(infohash, pstate)
-                    self.session.lm.rawserver.add_task(checkpoint, 0)
-
-                if alert_type == 'file_renamed_alert':
-                    if os.path.exists(self.unwanteddir_abs) and not os.listdir(self.unwanteddir_abs) and all(self.handle.file_priorities()):
-                        os.rmdir(self.unwanteddir_abs)
-
-                if alert_type == 'torrent_checked_alert' and self.pause_after_next_hashcheck:
-                    self.handle.pause()
-                    self.pause_after_next_hashcheck = False
-                    self.dlstate = DLSTATUS_STOPPED
+                    self.on_metadata_received_alert(alert)
+                elif alert_type == 'file_renamed_alert':
+                    self.on_file_renamed_alert(alert)
+                elif alert_type == 'performance_alert':
+                    self.on_performance_alert(alert)
+                elif alert_type == 'torrent_checked_alert':
+                    self.on_torrent_checked_alert(alert)
                 else:
-                    if alert_type == 'torrent_paused_alert':
-                        self.dlstate = DLSTATUS_STOPPED_ON_ERROR if status.error else DLSTATUS_STOPPED
-                    else:
-                        self.dlstate = self.dlstates[status.state] if not status.paused else DLSTATUS_STOPPED
-
-                self.error = unicode(status.error) if status.error else None
-                self.length = float(status.total_wanted)
-                self.progress = status.progress
-                self.curspeeds[DOWNLOAD] = float(status.download_payload_rate) if self.dlstate not in [DLSTATUS_STOPPED, DLSTATUS_STOPPED] else 0.0
-                self.curspeeds[UPLOAD] = float(status.upload_payload_rate) if self.dlstate not in [DLSTATUS_STOPPED, DLSTATUS_STOPPED] else 0.0
-                self.all_time_upload = status.all_time_upload
-                self.all_time_download = status.all_time_download
-                self.finished_time = status.finished_time
+                    self.update_lt_stats()                 
+
+    def on_metadata_received_alert(self, alert):
+        self.metadata = {'info': lt.bdecode(self.handle.get_torrent_info().metadata())}
+
+        trackers = [tracker['url'] for tracker in self.handle.trackers()]
+        if trackers:
+            if len(trackers) > 1:
+                self.metadata["announce-list"] = [trackers]
+            else:
+                self.metadata["announce"] = trackers[0]
+
+        self.tdef = TorrentDef.load_from_dict(self.metadata)
+        self.orig_files = [torrent_file.path for torrent_file in lt.torrent_info(self.metadata).files()]
+        self.set_files()
+
+        if self.session.lm.rtorrent_handler:
+            self.session.lm.rtorrent_handler.save_torrent(self.tdef)
+        elif self.session.lm.torrent_db:
+            self.session.lm.torrent_db.addExternalTorrent(self.tdef, source = '', extra_info = {'status':'good'}, commit = True)
+
+        self.checkpoint()
+
+    def on_file_renamed_alert(self, alert):
+        if os.path.exists(self.unwanteddir_abs) and not os.listdir(self.unwanteddir_abs) and all(self.handle.file_priorities()):
+            os.rmdir(self.unwanteddir_abs)
+
+    def on_performance_alert(self, alert):
+        # When the send buffer watermark is too low, double the buffer size to a maximum of 50MiB. This is the same mechanism as Deluge uses.
+        if alert.message().endswith("send buffer watermark too low (upload rate will suffer)"): 
+            settings = self.ltmgr.ltsession.settings()
+            if settings.send_buffer_watermark <= 26214400:
+                print >> sys.stderr, "LibtorrentDownloadImpl: setting send_buffer_watermark to", 2 * settings.send_buffer_watermark
+                settings.send_buffer_watermark = 2 * settings.send_buffer_watermark 
+                self.ltmgr.ltsession.set_settings(settings)
+        # When the write cache is too small, double the buffer size to a maximum of 64MiB. Again, this is the same mechanism as Deluge uses.
+        elif alert.message().endswith("max outstanding disk writes reached"):
+            settings = self.ltmgr.ltsession.settings()
+            if settings.max_queued_disk_bytes <= 33554432:
+                print >> sys.stderr, "LibtorrentDownloadImpl: setting max_queued_disk_bytes to", 2 * settings.max_queued_disk_bytes
+                settings.max_queued_disk_bytes = 2 * settings.max_queued_disk_bytes 
+                self.ltmgr.ltsession.set_settings(settings) 
+
+    def on_torrent_checked_alert(self, alert):
+        if self.pause_after_next_hashcheck:
+            self.pause_after_next_hashcheck = False
+            self.handle.pause()
+        if self.checkpoint_after_next_hashcheck:
+            self.checkpoint_after_next_hashcheck = False
+            self.checkpoint()
+
+    def update_lt_stats(self):
+        status = self.handle.status()
+        self.dlstate = self.dlstates[status.state] if not status.paused else DLSTATUS_STOPPED
+        self.dlstate = DLSTATUS_STOPPED_ON_ERROR if self.dlstate == DLSTATUS_STOPPED and status.error else self.dlstate
+        self.error = unicode(status.error) if status.error else None
+        self.length = float(status.total_wanted)
+        self.progress = status.progress
+        self.curspeeds[DOWNLOAD] = float(status.download_payload_rate) if self.dlstate not in [DLSTATUS_STOPPED, DLSTATUS_STOPPED] else 0.0
+        self.curspeeds[UPLOAD] = float(status.upload_payload_rate) if self.dlstate not in [DLSTATUS_STOPPED, DLSTATUS_STOPPED] else 0.0
+        self.all_time_upload = status.all_time_upload
+        self.all_time_download = status.all_time_download
+        self.finished_time = status.finished_time
 
     def set_files(self):
         metainfo = self.tdef.get_metainfo()
@@ -423,6 +470,7 @@ def force_recheck(self):
             if self.handle is not None and not isinstance(self.tdef, TorrentDefNoMetainfo):
                 if self.dlstate == DLSTATUS_STOPPED:
                     self.pause_after_next_hashcheck = True
+                self.checkpoint_after_next_hashcheck = True
                 self.handle.resume()
                 self.handle.force_recheck()
 
@@ -567,7 +615,7 @@ def network_get_state(self,usercallback, getpeerlist, sessioncalling = False):
             if self.handle is None:
                 if DEBUG:
                     print >> sys.stderr, "LibtorrentDownloadImpl: network_get_state: Download not running"
-                ds = DownloadState(self, DLSTATUS_STOPPED, self.error, self.progressbeforestop)
+                ds = DownloadState(self, DLSTATUS_WAITING4HASHCHECK, self.error, self.progressbeforestop)
             else:
                 (status, stats, seeding_stats, logmsgs) = self.network_get_stats(getpeerlist)
                 ds = DownloadState(self, status, self.error, self.get_progress(), stats = stats, seeding_stats = seeding_stats, filepieceranges = self.filepieceranges, logmsgs = logmsgs)
@@ -613,8 +661,7 @@ def network_stop(self, removestate, removecontent):
                     self.handle = None
                 else:
                     self.handle.pause()
-                    pstate['engineresumedata'] = self.handle.write_resume_data() if getattr(self.handle.status(), 'has_metadata', False) else None
-                    self.dlstate = DLSTATUS_STOPPED
+                    pstate['engineresumedata'] = self.handle.write_resume_data() if isinstance(self.tdef, TorrentDef) else None
                 self.pstate_for_restart = pstate
             else:
                 # This method is also called at Session shutdown, where one may
@@ -659,6 +706,7 @@ def restart(self, initialdlstatus = None):
         # Called by any thread 
         if DEBUG:
             print >>sys.stderr,"LibtorrentDownloadImpl: restart:", self.tdef.get_name()
+
         with self.dllock:
             if self.handle is None:
                 self.error = None
@@ -728,7 +776,9 @@ def get_ext(filename):
 
     def checkpoint(self):
         """ Called by any thread """
-        self.network_checkpoint()
+        (infohash, pstate) = self.network_checkpoint()
+        checkpoint = lambda : self.session.lm.save_download_pstate(infohash, pstate)
+        self.session.lm.rawserver.add_task(checkpoint, 0)
 
     def network_checkpoint(self):
         """ Called by network thread """
@@ -738,7 +788,7 @@ def network_checkpoint(self):
             if self.handle == None:
                 if self.pstate_for_restart is not None:
                     resdata = self.pstate_for_restart['engineresumedata']
-            elif getattr(self.handle.status(), 'has_metadata', False):
+            elif isinstance(self.tdef, TorrentDef):
                 resdata = self.handle.write_resume_data()
             pstate['engineresumedata'] = resdata
             return (self.tdef.get_infohash(), pstate)

Tribler/Core/Libtorrent/LibtorrentMgr.py

@@ -28,11 +28,9 @@ def __init__(self, trsession, utility):
         settings = lt.session_settings()
         settings.user_agent = 'Tribler/' + version_id
         fingerprint = ['TL'] + map(int, version_id.split('.')) + [0]
-        self.ltsession = lt.session(lt.fingerprint(*fingerprint))
+        # Workaround for libtorrent 0.16.3 segfault (see https://code.google.com/p/libtorrent/issues/detail?id=369)
+        self.ltsession = lt.session(lt.fingerprint(*fingerprint), flags = 1)
         self.ltsession.set_settings(settings)
-        self.ltsession.add_extension(lt.create_ut_metadata_plugin)
-        self.ltsession.add_extension(lt.create_ut_pex_plugin)
-        self.ltsession.add_extension(lt.create_smart_ban_plugin)
         self.ltsession.set_alert_mask(lt.alert.category_t.stats_notification |
                                       lt.alert.category_t.error_notification |
                                       lt.alert.category_t.status_notification |
@@ -57,6 +55,7 @@ def __init__(self, trsession, utility):
         self.torlock = NoDispersyRLock()
         self.torrents = {}
         self.trsession.lm.rawserver.add_task(self.process_alerts, 1)
+        self.trsession.lm.rawserver.add_task(self.reachability_check, 1)
 
     def getInstance(*args, **kw):
         if LibtorrentMgr.__single is None:
@@ -169,3 +168,9 @@ def process_alerts(self):
                         print >> sys.stderr, "LibtorrentMgr: alert for invalid torrent"
                 alert = self.ltsession.pop_alert()
             self.trsession.lm.rawserver.add_task(self.process_alerts, 1)
+
+    def reachability_check(self):
+        if self.ltsession and self.ltsession.status().has_incoming_connections:
+            self.trsession.lm.dialback_reachable_callback()
+        else:
+            self.trsession.lm.rawserver.add_task(self.reachability_check, 10)

Tribler/Core/RemoteTorrentHandler.py

@@ -494,6 +494,7 @@ def __init__(self, remote_th, magnet_requester, session, prio):
         defaultDLConfig = DefaultDownloadStartupConfig.getInstance()
         self.dscfg = defaultDLConfig.copy()
         self.dscfg.set_dest_dir(session.get_torrent_collecting_dir())
+        self.dscfg.set_swift_meta_dir(session.get_torrent_collecting_dir())
 
 
     def doFetch(self, hash, candidates):
@@ -689,6 +690,7 @@ def __init__(self, remote_th, session):
         defaultDLConfig = DefaultDownloadStartupConfig.getInstance()
         self.dscfg = defaultDLConfig.copy()
         self.dscfg.set_dest_dir(session.get_torrent_collecting_dir())
+        self.dscfg.set_swift_meta_dir(session.get_torrent_collecting_dir())
 
     def doFetch(self, hash_tuple, candidates):
         roothash, infohash = hash_tuple

Tribler/Core/Swift/SwiftDownloadImpl.py

@@ -37,6 +37,8 @@
 from Tribler.Core.simpledefs import *
 from Tribler.Core.DownloadState import *
 from Tribler.Core.Swift.SwiftDownloadRuntimeConfig import SwiftDownloadRuntimeConfig
+from Tribler.Core.DownloadConfig import get_default_dest_dir
+import shutil
 
 # ARNOSMPTODO: MODIFY WITH cmdgw.cpp::CMDGW_PREBUFFER_BYTES_AS_LAYER
 # Send PLAY after receiving 2^layer * 1024 bytes
@@ -153,6 +155,25 @@ def network_create_engine_wrapper(self,lm_network_engine_wrapper_created_callbac
         if self.get_mode() == DLMODE_VOD:        
             self.lm_network_vod_event_callback = lm_network_vod_event_callback
 
+        if not self.dlconfig.has_key('swiftmetadir') and not os.path.isdir(self.get_dest_dir()):
+            # We must be dealing with a checkpoint from a previous release (<6.1.0). Move the swift metadata to the right directory.
+            metadir = os.path.join(get_default_dest_dir(), STATEDIR_SWIFTRESEED_DIR)
+            self.set_swift_meta_dir(metadir)
+            if not os.path.exists(metadir):
+                os.makedirs(metadir)
+
+            is_multifile = self.get_dest_dir().endswith("." + self.get_def().get_roothash_as_hex())
+            path_old = self.get_dest_dir()
+            path_new = os.path.join(metadir, self.get_def().get_roothash_as_hex() if is_multifile else os.path.split(self.get_dest_dir())[1])
+            try:
+                if is_multifile:
+                    shutil.move(path_old, path_new + '.mfspec')
+                    self.dlconfig['saveas'] = os.path.split(self.get_dest_dir())[0]
+                shutil.move(path_old + '.mhash', path_new + '.mhash')
+                shutil.move(path_old + '.mbinmap', path_new + '.mbinmap')
+            except:
+                print_exc()
+
         # Synchronous: starts process if needed
         self.sp = self.session.lm.spm.get_or_create_sp(self.session.get_swift_working_dir(),self.session.get_torrent_collecting_dir(),self.get_swift_listen_port(), self.get_swift_httpgw_listen_port(), self.get_swift_cmdgw_listen_port() )
         self.sp.start_download(self)