From c89595d9293f5ecce212fd32c38be25543e27619 Mon Sep 17 00:00:00 2001
From: Isak Samsten <isak@samsten.se>
Date: Thu, 13 Aug 2020 13:44:03 +0200
Subject: [PATCH 1/3] Enable cross-seed deletion/retention

Cross-seeds are detected by torrents that share the same root path

Cross-seed handling is enabled if the config `delete_data` is set to
`True`. If no data is deleted the cross-seed is handled by another
strategy. If `delete_data` is set to `True` torrents seeding the same
data as a torrent marked for deletion will fail.

If enabled, deletion is by defailt not propagated to the cross-seeded
torrents, instead the torrent marked for removal is flagged as
soft-remove, meaning that the torrent will be deleted but not the
data.

By changing the flag `remove_cross_seeds` in the strategy
configuration to `all_trackers` or a list of tracker host names
deletion will propagate to either all or the selected
trackers. Torrents included in the proagation will also be deleted,
even if not included by the filter.

Support has been added to:
 * deluge
 * qBittorrent APIv2
 * Transmission

A new field has been added to `Torrent#base_path`
---
 autoremovetorrents/client/deluge.py       |  3 +
 autoremovetorrents/client/qbittorrent.py  |  8 +++
 autoremovetorrents/client/transmission.py |  4 +-
 autoremovetorrents/client/utorrent.py     |  1 +
 autoremovetorrents/filter/tracker.py      |  2 +-
 autoremovetorrents/strategy.py            | 77 +++++++++++++++++++++--
 autoremovetorrents/task.py                | 19 +++---
 7 files changed, 100 insertions(+), 14 deletions(-)

diff --git a/autoremovetorrents/client/deluge.py b/autoremovetorrents/client/deluge.py
index dfba8fa..f6da6dc 100644
--- a/autoremovetorrents/client/deluge.py
+++ b/autoremovetorrents/client/deluge.py
@@ -1,4 +1,5 @@
 import time
+from pathlib import Path
 from deluge_client import DelugeRPCClient
 from deluge_client.client import DelugeClientException
 from ..torrent import Torrent
@@ -86,6 +87,7 @@ def torrents_list(self):
             'total_uploaded',
             'trackers',
             'upload_payload_rate',
+            'files'
         ])
         # Save properties to cache
         self._torrent_cache = torrent_list
@@ -129,6 +131,7 @@ def torrent_properties(self, torrent_hash):
             # Set the last active time of those never active torrents to timestamp 0
             torrent_obj.last_activity = torrent['time_since_transfer'] if torrent['time_since_transfer'] > 0 else 0
         torrent_obj.progress = torrent['progress'] / 100 # Accept Range: 0-1
+        torrent_obj.root_path = Path(torrent['files'][0]['path']).parts[0]
 
         return torrent_obj
 
diff --git a/autoremovetorrents/client/qbittorrent.py b/autoremovetorrents/client/qbittorrent.py
index 9df021c..4805353 100644
--- a/autoremovetorrents/client/qbittorrent.py
+++ b/autoremovetorrents/client/qbittorrent.py
@@ -1,6 +1,7 @@
 #-*- coding:utf-8 -*-
 import requests
 import time
+from pathlib import Path
 from ..torrent import Torrent
 from ..torrentstatus import TorrentStatus
 from ..exception.loginfailure import LoginFailure
@@ -93,6 +94,10 @@ def torrent_list(self):
         # Get torrent's generic properties
         def torrent_generic_properties(self, torrent_hash):
             return self._session.get(self._host+'/api/v2/torrents/properties', params={'hash': torrent_hash})
+
+        # Get torrent's files
+        def torrent_files(self, torrent_hash):
+            return self._session.get(self._host+'/api/v2/torrents/files', params={'hash': torrent_hash})
         
         # Get torrent's tracker
         def torrent_trackers(self, torrent_hash):
@@ -167,6 +172,8 @@ def torrent_properties(self, torrent_hash):
                 # Get other information
                 properties = self._request_handler.torrent_generic_properties(torrent_hash).json()
                 trackers = self._request_handler.torrent_trackers(torrent_hash).json()
+                files = self._request_handler.torrent_files(torrent_hash).json()
+
                 # Create torrent object
                 torrent_obj = Torrent()
                 torrent_obj.hash = torrent['hash']
@@ -197,6 +204,7 @@ def torrent_properties(self, torrent_hash):
                 if 'last_activity' in torrent:
                     torrent_obj.last_activity = torrent['last_activity']
                 torrent_obj.progress = torrent['progress']
+                torrent_obj.root_path = Path(files[0]['name']).parts[0]
 
                 return torrent_obj
 
diff --git a/autoremovetorrents/client/transmission.py b/autoremovetorrents/client/transmission.py
index 8020886..cfea32c 100644
--- a/autoremovetorrents/client/transmission.py
+++ b/autoremovetorrents/client/transmission.py
@@ -1,5 +1,6 @@
 #-*- coding:utf-8 -*-
 import requests
+from pathlib import Path
 from ..torrent import Torrent
 from ..torrentstatus import TorrentStatus
 from ..exception.connectionfailure import ConnectionFailure
@@ -99,6 +100,7 @@ def torrent_properties(self, torrent_hash):
                     'downloadedEver',
                     'secondsDownloading',
                     'percentDone',
+                    'files'
                 ]}
             )
         if len(result['torrents']) == 0: # No such torrent
@@ -128,7 +130,7 @@ def torrent_properties(self, torrent_hash):
         torrent_obj.average_upload_speed = torrent['uploadedEver'] / torrent['secondsSeeding'] if torrent['secondsSeeding'] != 0 else 0
         torrent_obj.average_download_speed = torrent['downloadedEver'] / torrent['secondsDownloading'] if torrent['secondsDownloading'] != 0 else 0
         torrent_obj.progress = torrent['percentDone']
-
+        torrent_obj.root_path = Path(torrent['files'][0]['name']).parts[0]
         return torrent_obj
 
     # Judge Torrent Status
diff --git a/autoremovetorrents/client/utorrent.py b/autoremovetorrents/client/utorrent.py
index aa21595..e0f4323 100644
--- a/autoremovetorrents/client/utorrent.py
+++ b/autoremovetorrents/client/utorrent.py
@@ -106,6 +106,7 @@ def torrent_properties(self, torrent_hash):
                 torrent_obj.leecher = torrent[13]
                 torrent_obj.connected_leecher = torrent[12]
                 torrent_obj.progress = torrent[4]
+                torrent_obj.root_path = None # unsupported
 
                 return torrent_obj
         # Not Found
diff --git a/autoremovetorrents/filter/tracker.py b/autoremovetorrents/filter/tracker.py
index 4a88d0a..18466c5 100644
--- a/autoremovetorrents/filter/tracker.py
+++ b/autoremovetorrents/filter/tracker.py
@@ -26,4 +26,4 @@ def apply(self, torrents):
                     hostname = urlparse_(tracker).hostname
                     if hostname in self._reject or tracker in self._reject:
                         rejects.add(torrent)
-        return accepts.difference(rejects) # Return their difference
\ No newline at end of file
+        return accepts.difference(rejects) # Return their difference
diff --git a/autoremovetorrents/strategy.py b/autoremovetorrents/strategy.py
index b7d5e0a..376e97d 100644
--- a/autoremovetorrents/strategy.py
+++ b/autoremovetorrents/strategy.py
@@ -1,5 +1,7 @@
 #-*- coding:utf-8 -*-
+from collections import defaultdict
 from . import logger
+from .compatibility.urlparse_ import urlparse_
 from .condition.avgdownloadspeed import AverageDownloadSpeedCondition
 from .condition.avguploadspeed import AverageUploadSpeedCondition
 from .condition.connectedleecher import ConnectedLeecherCondition
@@ -26,7 +28,7 @@
 from .filter.tracker import TrackerFilter
 
 class Strategy(object):
-    def __init__(self, name, conf):
+    def __init__(self, name, conf, delete_data):
         # Logger
         self._logger = logger.Logger.register(__name__)
 
@@ -36,9 +38,19 @@ def __init__(self, name, conf):
         # Configuration
         self._conf = conf
 
+        self._delete_data = delete_data
+        # Default to not deleting cross-seeds
+        self._delete_cross_seeds = conf['remove_cross_seeds'] if 'remove_cross_seeds' in conf else []
+
+        # if all_trackers remove any cross-seeds. All other terms are
+        # considered tracker hostnames.
+        if self._delete_cross_seeds != "all_trackers" and type(self._delete_cross_seeds) != list:
+            self._delete_cross_seeds = [self._delete_cross_seeds]
+
         # Results
         self.remain_list = set()
         self.remove_list = set()
+        self.soft_remove_list = set()
 
         # Filter ALL
         self._all_categories = conf['all_categories'] if 'all_categories' in conf \
@@ -109,6 +121,52 @@ def _apply_conditions(self):
                 self.remain_list = cond.remain
                 self.remove_list.update(cond.remove)
 
+    def _remove_cross_seeds(self, torrents):
+        delete_hash = {torrent.hash for torrent in self.remove_list}
+        remain = defaultdict(list)
+        self._logger.info("Searching for cross-seeds to delete")
+        for torrent in torrents:
+            if torrent.hash not in delete_hash:
+                remain[torrent.root_path].append(torrent)
+
+        remove_list = set()
+        soft_remove_list = set()
+        for torrent in self.remove_list:
+            if torrent.root_path in remain:
+                cross_seeds = remain[torrent.root_path]
+                marked_cross_seeds = set()
+                for cross_seed in cross_seeds:
+                    allowed_trackers = [urlparse_(tracker).hostname for tracker in cross_seed.tracker]
+                    if self._delete_cross_seeds == "all_trackers" or any(allowed_tracker in self._delete_cross_seeds for allowed_tracker in allowed_trackers):
+                        self._logger.info("Remove cross-seed: %s (trackers: %s)" % (cross_seed.name, ",".join(allowed_trackers)))
+                        marked_cross_seeds.add(cross_seed)
+                    else:
+                        self._logger.info("Retain cross-seed: %s (trackers: %s)" % (cross_seed.name, ",".join(allowed_trackers)))
+
+                if len(marked_cross_seeds) == len(cross_seeds):
+                    # All cross-seededed torrents are marked for
+                    # deletion we can safely allow the data to be
+                    # deleted
+                    #
+                    # TODO: change remove_list to soft_remove_list
+                    # (the data is removed by the original torrent)
+                    remove_list.update(marked_cross_seeds)
+                    remove_list.add(torrent)
+                else:
+                    # Only some of the cross-seeded torrents torrents
+                    # have been marked for deletion. We cannot allow
+                    # the torrents to be deleted from disk or the
+                    # cross-seed will fail
+                    soft_remove_list.update(marked_cross_seeds)
+                    soft_remove_list.add(torrent)
+            else:
+                # Torrent is not cross-seeded, we can safely remove
+                # the data
+                remove_list.add(torrent)
+
+        self.remove_list = remove_list
+        self.soft_remove_list = soft_remove_list
+
     # Execute this strategy
     def execute(self, torrents):
         self._logger.info('Running strategy %s...' % self._name)
@@ -117,10 +175,21 @@ def execute(self, torrents):
         self._apply_filters()
         # Apply Conditions
         self._apply_conditions()
+
+        # If we don't remove data, the cross-seeds should be handled
+        # by another strategy. If we delete the data we should also
+        # delete the cross-seed
+        if self._delete_data:
+            self._remove_cross_seeds(torrents)
+
         # Print remove list
-        self._logger.info("Total: %d torrent(s). %d torrent(s) can be removed." %
-            (len(self.remain_list)+len(self.remove_list), len(self.remove_list)))
+        self._logger.info("Total: %d torrent(s). %d torrent(s) can be removed. %d torrent(s) can be safe removed (without data)." %
+            (len(self.remain_list)+len(self.remove_list), len(self.remove_list), len(self.soft_remove_list)))
         if len(self.remove_list) > 0:
             self._logger.info('To be deleted:')
             for torrent in self.remove_list:
-                self._logger.info(torrent)
\ No newline at end of file
+                self._logger.info(torrent)
+        if len(self.soft_remove_list) > 0:
+            self._logger.info('To be safe-deleted (without data):')
+            for torrent in self.soft_remove_list:
+                self._logger.info(torrent)
diff --git a/autoremovetorrents/task.py b/autoremovetorrents/task.py
index f42b7a4..faddc9b 100644
--- a/autoremovetorrents/task.py
+++ b/autoremovetorrents/task.py
@@ -41,6 +41,7 @@ def __init__(self, name, conf, remove_torrents = True):
         # Torrents
         self._torrents = set()
         self._remove = set()
+        self._soft_remove = set()
 
         # Allow removing specified torrents
         if 'force_delete' in conf:
@@ -91,27 +92,28 @@ def _get_torrents(self):
     # Apply strategies
     def _apply_strategies(self):
         for strategy_name in self._strategies:
-            strategy = Strategy(strategy_name, self._strategies[strategy_name])
+            strategy = Strategy(strategy_name, self._strategies[strategy_name], self._delete_data)
             strategy.execute(self._torrents)
             self._remove.update(strategy.remove_list)
+            self._soft_remove.update(strategy.soft_remove_list)
 
     # Remove torrents
-    def _remove_torrents(self):
+    def _remove_torrents(self, remove, delete_data):
         # Bulid a dict to store torrent hashes and names which to be deleted
         delete_list = {}
-        for torrent in self._remove:
+        for torrent in remove:
             delete_list[torrent.hash] = torrent.name
         # Run deletion
-        success, failed = self._client.remove_torrents([hash_ for hash_ in delete_list], self._delete_data)
+        success, failed = self._client.remove_torrents([hash_ for hash_ in delete_list], delete_data)
         # Output logs
         for hash_ in success:
             self._logger.info(
-                'The torrent %s and its data have been removed.' if self._delete_data \
+                'The torrent %s and its data have been removed.' if delete_data \
                 else 'The torrent %s has been removed.',
                 delete_list[hash_]
             )
         for torrent in failed:
-            self._logger.error('The torrent %s and its data cannot be removed. Reason: %s' if self._delete_data \
+            self._logger.error('The torrent %s and its data cannot be removed. Reason: %s' if delete_data \
                 else 'The torrent %s cannot be removed. Reason: %s',
                 delete_list[torrent['hash']], torrent['reason']
             )
@@ -123,7 +125,8 @@ def execute(self):
         self._get_torrents()
         self._apply_strategies()
         if self._enabled_remove:
-            self._remove_torrents()
+            self._remove_torrents(self._remove, self._delete_data)
+            self._remove_torrents(self._soft_remove, False)
 
     # Get remaining torrents (for tester)
     def get_remaining_torrents(self):
@@ -131,4 +134,4 @@ def get_remaining_torrents(self):
 
     # Get removed torrents (for tester)
     def get_removed_torrents(self):
-        return self._remove
\ No newline at end of file
+        return self._remove

From c2736cc803bd1d070555a4dfb06f322497056ae8 Mon Sep 17 00:00:00 2001
From: Isak Samsten <isak@samsten.se>
Date: Thu, 13 Aug 2020 15:16:17 +0200
Subject: [PATCH 2/3] Change to `os.path.dirname` instead to support Python 2.7

---
 autoremovetorrents/client/deluge.py       | 4 ++--
 autoremovetorrents/client/qbittorrent.py  | 3 +--
 autoremovetorrents/client/transmission.py | 4 ++--
 3 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/autoremovetorrents/client/deluge.py b/autoremovetorrents/client/deluge.py
index f6da6dc..ee8ae03 100644
--- a/autoremovetorrents/client/deluge.py
+++ b/autoremovetorrents/client/deluge.py
@@ -1,5 +1,5 @@
 import time
-from pathlib import Path
+import os
 from deluge_client import DelugeRPCClient
 from deluge_client.client import DelugeClientException
 from ..torrent import Torrent
@@ -131,7 +131,7 @@ def torrent_properties(self, torrent_hash):
             # Set the last active time of those never active torrents to timestamp 0
             torrent_obj.last_activity = torrent['time_since_transfer'] if torrent['time_since_transfer'] > 0 else 0
         torrent_obj.progress = torrent['progress'] / 100 # Accept Range: 0-1
-        torrent_obj.root_path = Path(torrent['files'][0]['path']).parts[0]
+        torrent_obj.root_path = os.path.dirname(torrent['files'][0]['path'])
 
         return torrent_obj
 
diff --git a/autoremovetorrents/client/qbittorrent.py b/autoremovetorrents/client/qbittorrent.py
index 4805353..2415b99 100644
--- a/autoremovetorrents/client/qbittorrent.py
+++ b/autoremovetorrents/client/qbittorrent.py
@@ -1,7 +1,6 @@
 #-*- coding:utf-8 -*-
 import requests
 import time
-from pathlib import Path
 from ..torrent import Torrent
 from ..torrentstatus import TorrentStatus
 from ..exception.loginfailure import LoginFailure
@@ -204,7 +203,7 @@ def torrent_properties(self, torrent_hash):
                 if 'last_activity' in torrent:
                     torrent_obj.last_activity = torrent['last_activity']
                 torrent_obj.progress = torrent['progress']
-                torrent_obj.root_path = Path(files[0]['name']).parts[0]
+                torrent_obj.root_path = os.path.dirname(files[0]['name'])
 
                 return torrent_obj
 
diff --git a/autoremovetorrents/client/transmission.py b/autoremovetorrents/client/transmission.py
index cfea32c..4f756c1 100644
--- a/autoremovetorrents/client/transmission.py
+++ b/autoremovetorrents/client/transmission.py
@@ -1,6 +1,6 @@
 #-*- coding:utf-8 -*-
 import requests
-from pathlib import Path
+import os
 from ..torrent import Torrent
 from ..torrentstatus import TorrentStatus
 from ..exception.connectionfailure import ConnectionFailure
@@ -130,7 +130,7 @@ def torrent_properties(self, torrent_hash):
         torrent_obj.average_upload_speed = torrent['uploadedEver'] / torrent['secondsSeeding'] if torrent['secondsSeeding'] != 0 else 0
         torrent_obj.average_download_speed = torrent['downloadedEver'] / torrent['secondsDownloading'] if torrent['secondsDownloading'] != 0 else 0
         torrent_obj.progress = torrent['percentDone']
-        torrent_obj.root_path = Path(torrent['files'][0]['name']).parts[0]
+        torrent_obj.root_path = os.path.dirname(torrent['files'][0]['name'])
         return torrent_obj
 
     # Judge Torrent Status

From f6445c3e5f3681cd950cb38c43632d78246a08e3 Mon Sep 17 00:00:00 2001
From: Isak Samsten <isak@samsten.se>
Date: Fri, 14 Aug 2020 15:25:30 +0200
Subject: [PATCH 3/3] Fix for empty base path

---
 autoremovetorrents/client/deluge.py            | 3 ++-
 autoremovetorrents/client/qbittorrent.py       | 3 ++-
 autoremovetorrents/client/transmission.py      | 3 ++-
 autoremovetorrents/compatibility/root_path_.py | 9 +++++++++
 4 files changed, 15 insertions(+), 3 deletions(-)
 create mode 100644 autoremovetorrents/compatibility/root_path_.py

diff --git a/autoremovetorrents/client/deluge.py b/autoremovetorrents/client/deluge.py
index ee8ae03..2ca3209 100644
--- a/autoremovetorrents/client/deluge.py
+++ b/autoremovetorrents/client/deluge.py
@@ -6,6 +6,7 @@
 from ..torrentstatus import TorrentStatus
 from ..exception.loginfailure import LoginFailure
 from ..exception.remotefailure import RemoteFailure
+from ..compatibility.root_path_ import root_path
 
 # Default port of Delgue
 DEFAULT_PORT = 58846
@@ -131,7 +132,7 @@ def torrent_properties(self, torrent_hash):
             # Set the last active time of those never active torrents to timestamp 0
             torrent_obj.last_activity = torrent['time_since_transfer'] if torrent['time_since_transfer'] > 0 else 0
         torrent_obj.progress = torrent['progress'] / 100 # Accept Range: 0-1
-        torrent_obj.root_path = os.path.dirname(torrent['files'][0]['path'])
+        torrent_obj.root_path = root_path(torrent['files'][0]['path'])
 
         return torrent_obj
 
diff --git a/autoremovetorrents/client/qbittorrent.py b/autoremovetorrents/client/qbittorrent.py
index 2415b99..a3bf5f9 100644
--- a/autoremovetorrents/client/qbittorrent.py
+++ b/autoremovetorrents/client/qbittorrent.py
@@ -6,6 +6,7 @@
 from ..exception.loginfailure import LoginFailure
 from ..exception.connectionfailure import ConnectionFailure
 from ..exception.incompatibleapi import IncompatibleAPIVersion
+from ..compatibility.root_path_ import root_path
 
 class qBittorrent(object):
     # API Handler for v1
@@ -203,7 +204,7 @@ def torrent_properties(self, torrent_hash):
                 if 'last_activity' in torrent:
                     torrent_obj.last_activity = torrent['last_activity']
                 torrent_obj.progress = torrent['progress']
-                torrent_obj.root_path = os.path.dirname(files[0]['name'])
+                torrent_obj.root_path = root_path(files[0]['name'])
 
                 return torrent_obj
 
diff --git a/autoremovetorrents/client/transmission.py b/autoremovetorrents/client/transmission.py
index 4f756c1..34422c6 100644
--- a/autoremovetorrents/client/transmission.py
+++ b/autoremovetorrents/client/transmission.py
@@ -7,6 +7,7 @@
 from ..exception.loginfailure import LoginFailure
 from ..exception.nosuchclient import NoSuchClient
 from ..exception.remotefailure import RemoteFailure
+from ..compatibility.root_path_ import root_path
 
 class Transmission(object):
     def __init__(self, host):
@@ -130,7 +131,7 @@ def torrent_properties(self, torrent_hash):
         torrent_obj.average_upload_speed = torrent['uploadedEver'] / torrent['secondsSeeding'] if torrent['secondsSeeding'] != 0 else 0
         torrent_obj.average_download_speed = torrent['downloadedEver'] / torrent['secondsDownloading'] if torrent['secondsDownloading'] != 0 else 0
         torrent_obj.progress = torrent['percentDone']
-        torrent_obj.root_path = os.path.dirname(torrent['files'][0]['name'])
+        torrent_obj.root_path = root_path(torrent['files'][0]['name'])
         return torrent_obj
 
     # Judge Torrent Status
diff --git a/autoremovetorrents/compatibility/root_path_.py b/autoremovetorrents/compatibility/root_path_.py
new file mode 100644
index 0000000..1381b2e
--- /dev/null
+++ b/autoremovetorrents/compatibility/root_path_.py
@@ -0,0 +1,9 @@
+def root_path(path):
+    try: # for Python 3
+        from pathlib import Path
+        return Path(path).parts[0]
+    except ImportError: # for Python 2.7
+        import os
+        dirname = os.path.dirname(path)
+        return dirname if dirname != '' else path
+