Enable cross-seed deletion/retention

autoremovetorrents/client/deluge.py

@@ -1,10 +1,12 @@
 import time
+import os
 from deluge_client import DelugeRPCClient
 from deluge_client.client import DelugeClientException
 from ..torrent import Torrent
 from ..torrentstatus import TorrentStatus
 from ..exception.loginfailure import LoginFailure
 from ..exception.remotefailure import RemoteFailure
+from ..compatibility.root_path_ import root_path
 
 # Default port of Delgue
 DEFAULT_PORT = 58846
@@ -86,6 +88,7 @@ def torrents_list(self):
             'total_uploaded',
             'trackers',
             'upload_payload_rate',
+            'files'
         ])
         # Save properties to cache
         self._torrent_cache = torrent_list
@@ -129,6 +132,7 @@ def torrent_properties(self, torrent_hash):
             # Set the last active time of those never active torrents to timestamp 0
             torrent_obj.last_activity = torrent['time_since_transfer'] if torrent['time_since_transfer'] > 0 else 0
         torrent_obj.progress = torrent['progress'] / 100 # Accept Range: 0-1
+        torrent_obj.root_path = root_path(torrent['files'][0]['path'])
 
         return torrent_obj
 

autoremovetorrents/client/qbittorrent.py

@@ -6,6 +6,7 @@
 from ..exception.loginfailure import LoginFailure
 from ..exception.connectionfailure import ConnectionFailure
 from ..exception.incompatibleapi import IncompatibleAPIVersion
+from ..compatibility.root_path_ import root_path
 
 class qBittorrent(object):
     # API Handler for v1
@@ -93,6 +94,10 @@ def torrent_list(self):
         # Get torrent's generic properties
         def torrent_generic_properties(self, torrent_hash):
             return self._session.get(self._host+'/api/v2/torrents/properties', params={'hash': torrent_hash})
+
+        # Get torrent's files
+        def torrent_files(self, torrent_hash):
+            return self._session.get(self._host+'/api/v2/torrents/files', params={'hash': torrent_hash})
 
         # Get torrent's tracker
         def torrent_trackers(self, torrent_hash):
@@ -167,6 +172,8 @@ def torrent_properties(self, torrent_hash):
                 # Get other information
                 properties = self._request_handler.torrent_generic_properties(torrent_hash).json()
                 trackers = self._request_handler.torrent_trackers(torrent_hash).json()
+                files = self._request_handler.torrent_files(torrent_hash).json()
+
                 # Create torrent object
                 torrent_obj = Torrent()
                 torrent_obj.hash = torrent['hash']
@@ -197,6 +204,7 @@ def torrent_properties(self, torrent_hash):
                 if 'last_activity' in torrent:
                     torrent_obj.last_activity = torrent['last_activity']
                 torrent_obj.progress = torrent['progress']
+                torrent_obj.root_path = root_path(files[0]['name'])
 
                 return torrent_obj
 

autoremovetorrents/client/transmission.py

@@ -1,11 +1,13 @@
 #-*- coding:utf-8 -*-
 import requests
+import os
 from ..torrent import Torrent
 from ..torrentstatus import TorrentStatus
 from ..exception.connectionfailure import ConnectionFailure
 from ..exception.loginfailure import LoginFailure
 from ..exception.nosuchclient import NoSuchClient
 from ..exception.remotefailure import RemoteFailure
+from ..compatibility.root_path_ import root_path
 
 class Transmission(object):
     def __init__(self, host):
@@ -99,6 +101,7 @@ def torrent_properties(self, torrent_hash):
                     'downloadedEver',
                     'secondsDownloading',
                     'percentDone',
+                    'files'
                 ]}
             )
         if len(result['torrents']) == 0: # No such torrent
@@ -128,7 +131,7 @@ def torrent_properties(self, torrent_hash):
         torrent_obj.average_upload_speed = torrent['uploadedEver'] / torrent['secondsSeeding'] if torrent['secondsSeeding'] != 0 else 0
         torrent_obj.average_download_speed = torrent['downloadedEver'] / torrent['secondsDownloading'] if torrent['secondsDownloading'] != 0 else 0
         torrent_obj.progress = torrent['percentDone']
-
+        torrent_obj.root_path = root_path(torrent['files'][0]['name'])
         return torrent_obj
 
     # Judge Torrent Status

autoremovetorrents/client/utorrent.py

@@ -106,6 +106,7 @@ def torrent_properties(self, torrent_hash):
                 torrent_obj.leecher = torrent[13]
                 torrent_obj.connected_leecher = torrent[12]
                 torrent_obj.progress = torrent[4]
+                torrent_obj.root_path = None # unsupported
 
                 return torrent_obj
         # Not Found

autoremovetorrents/compatibility/root_path_.py

@@ -0,0 +1,9 @@
+def root_path(path):
+    try: # for Python 3
+        from pathlib import Path
+        return Path(path).parts[0]
+    except ImportError: # for Python 2.7
+        import os
+        dirname = os.path.dirname(path)
+        return dirname if dirname != '' else path
+

autoremovetorrents/filter/tracker.py

@@ -26,4 +26,4 @@ def apply(self, torrents):
                     hostname = urlparse_(tracker).hostname
                     if hostname in self._reject or tracker in self._reject:
                         rejects.add(torrent)
-        return accepts.difference(rejects) # Return their difference
+        return accepts.difference(rejects) # Return their difference

autoremovetorrents/strategy.py

@@ -1,5 +1,7 @@
 #-*- coding:utf-8 -*-
+from collections import defaultdict
 from . import logger
+from .compatibility.urlparse_ import urlparse_
 from .condition.avgdownloadspeed import AverageDownloadSpeedCondition
 from .condition.avguploadspeed import AverageUploadSpeedCondition
 from .condition.connectedleecher import ConnectedLeecherCondition
@@ -26,7 +28,7 @@
 from .filter.tracker import TrackerFilter
 
 class Strategy(object):
-    def __init__(self, name, conf):
+    def __init__(self, name, conf, delete_data):
         # Logger
         self._logger = logger.Logger.register(__name__)
 
@@ -36,9 +38,19 @@ def __init__(self, name, conf):
         # Configuration
         self._conf = conf
 
+        self._delete_data = delete_data
+        # Default to not deleting cross-seeds
+        self._delete_cross_seeds = conf['remove_cross_seeds'] if 'remove_cross_seeds' in conf else []
+
+        # if all_trackers remove any cross-seeds. All other terms are
+        # considered tracker hostnames.
+        if self._delete_cross_seeds != "all_trackers" and type(self._delete_cross_seeds) != list:
+            self._delete_cross_seeds = [self._delete_cross_seeds]
+
         # Results
         self.remain_list = set()
         self.remove_list = set()
+        self.soft_remove_list = set()
 
         # Filter ALL
         self._all_categories = conf['all_categories'] if 'all_categories' in conf \
@@ -109,6 +121,52 @@ def _apply_conditions(self):
                 self.remain_list = cond.remain
                 self.remove_list.update(cond.remove)
 
+    def _remove_cross_seeds(self, torrents):
+        delete_hash = {torrent.hash for torrent in self.remove_list}
+        remain = defaultdict(list)
+        self._logger.info("Searching for cross-seeds to delete")
+        for torrent in torrents:
+            if torrent.hash not in delete_hash:
+                remain[torrent.root_path].append(torrent)
+
+        remove_list = set()
+        soft_remove_list = set()
+        for torrent in self.remove_list:
+            if torrent.root_path in remain:
+                cross_seeds = remain[torrent.root_path]
+                marked_cross_seeds = set()
+                for cross_seed in cross_seeds:
+                    allowed_trackers = [urlparse_(tracker).hostname for tracker in cross_seed.tracker]
+                    if self._delete_cross_seeds == "all_trackers" or any(allowed_tracker in self._delete_cross_seeds for allowed_tracker in allowed_trackers):
+                        self._logger.info("Remove cross-seed: %s (trackers: %s)" % (cross_seed.name, ",".join(allowed_trackers)))
+                        marked_cross_seeds.add(cross_seed)
+                    else:
+                        self._logger.info("Retain cross-seed: %s (trackers: %s)" % (cross_seed.name, ",".join(allowed_trackers)))
+
+                if len(marked_cross_seeds) == len(cross_seeds):
+                    # All cross-seededed torrents are marked for
+                    # deletion we can safely allow the data to be
+                    # deleted
+                    #
+                    # TODO: change remove_list to soft_remove_list
+                    # (the data is removed by the original torrent)
+                    remove_list.update(marked_cross_seeds)
+                    remove_list.add(torrent)
+                else:
+                    # Only some of the cross-seeded torrents torrents
+                    # have been marked for deletion. We cannot allow
+                    # the torrents to be deleted from disk or the
+                    # cross-seed will fail
+                    soft_remove_list.update(marked_cross_seeds)
+                    soft_remove_list.add(torrent)
+            else:
+                # Torrent is not cross-seeded, we can safely remove
+                # the data
+                remove_list.add(torrent)
+
+        self.remove_list = remove_list
+        self.soft_remove_list = soft_remove_list
+
     # Execute this strategy
     def execute(self, torrents):
         self._logger.info('Running strategy %s...' % self._name)
@@ -117,10 +175,21 @@ def execute(self, torrents):
         self._apply_filters()
         # Apply Conditions
         self._apply_conditions()
+
+        # If we don't remove data, the cross-seeds should be handled
+        # by another strategy. If we delete the data we should also
+        # delete the cross-seed
+        if self._delete_data:
+            self._remove_cross_seeds(torrents)
+
         # Print remove list
-        self._logger.info("Total: %d torrent(s). %d torrent(s) can be removed." %
-            (len(self.remain_list)+len(self.remove_list), len(self.remove_list)))
+        self._logger.info("Total: %d torrent(s). %d torrent(s) can be removed. %d torrent(s) can be safe removed (without data)." %
+            (len(self.remain_list)+len(self.remove_list), len(self.remove_list), len(self.soft_remove_list)))
         if len(self.remove_list) > 0:
             self._logger.info('To be deleted:')
             for torrent in self.remove_list:
-                self._logger.info(torrent)
+                self._logger.info(torrent)
+        if len(self.soft_remove_list) > 0:
+            self._logger.info('To be safe-deleted (without data):')
+            for torrent in self.soft_remove_list:
+                self._logger.info(torrent)

autoremovetorrents/task.py

@@ -41,6 +41,7 @@ def __init__(self, name, conf, remove_torrents = True):
         # Torrents
         self._torrents = set()
         self._remove = set()
+        self._soft_remove = set()
 
         # Allow removing specified torrents
         if 'force_delete' in conf:
@@ -91,27 +92,28 @@ def _get_torrents(self):
     # Apply strategies
     def _apply_strategies(self):
         for strategy_name in self._strategies:
-            strategy = Strategy(strategy_name, self._strategies[strategy_name])
+            strategy = Strategy(strategy_name, self._strategies[strategy_name], self._delete_data)
             strategy.execute(self._torrents)
             self._remove.update(strategy.remove_list)
+            self._soft_remove.update(strategy.soft_remove_list)
 
     # Remove torrents
-    def _remove_torrents(self):
+    def _remove_torrents(self, remove, delete_data):
         # Bulid a dict to store torrent hashes and names which to be deleted
         delete_list = {}
-        for torrent in self._remove:
+        for torrent in remove:
             delete_list[torrent.hash] = torrent.name
         # Run deletion
-        success, failed = self._client.remove_torrents([hash_ for hash_ in delete_list], self._delete_data)
+        success, failed = self._client.remove_torrents([hash_ for hash_ in delete_list], delete_data)
         # Output logs
         for hash_ in success:
             self._logger.info(
-                'The torrent %s and its data have been removed.' if self._delete_data \
+                'The torrent %s and its data have been removed.' if delete_data \
                 else 'The torrent %s has been removed.',
                 delete_list[hash_]
             )
         for torrent in failed:
-            self._logger.error('The torrent %s and its data cannot be removed. Reason: %s' if self._delete_data \
+            self._logger.error('The torrent %s and its data cannot be removed. Reason: %s' if delete_data \
                 else 'The torrent %s cannot be removed. Reason: %s',
                 delete_list[torrent['hash']], torrent['reason']
             )
@@ -123,12 +125,13 @@ def execute(self):
         self._get_torrents()
         self._apply_strategies()
         if self._enabled_remove:
-            self._remove_torrents()
+            self._remove_torrents(self._remove, self._delete_data)
+            self._remove_torrents(self._soft_remove, False)
 
     # Get remaining torrents (for tester)
     def get_remaining_torrents(self):
         return self._torrents
 
     # Get removed torrents (for tester)
     def get_removed_torrents(self):
-        return self._remove
+        return self._remove