Resource storage older than X days deletion (#392)

* add parameter for resource_storage deletion to delete only files older than x days

Co-authored-by: anikaweinmann <[email protected]>

src/actinia_core/processing/actinia_processing/persistent/resource_storage_management.py

@@ -36,7 +36,7 @@
 
 
 __license__ = "GPLv3"
-__author__ = "Sören Gebbert"
+__author__ = "Sören Gebbert, Anika Weinmann"
 __copyright__ = (
     "Copyright 2016-2022, Sören Gebbert and mundialis GmbH & Co. KG"
 )
@@ -47,10 +47,12 @@ class ResourceStorageDelete(PersistentProcessing):
     """Delete the user specific resource directory"""
 
     def __init__(self, *args):
-        PersistentProcessing.__init__(self, *args)
+        rdc = args[0]
+        PersistentProcessing.__init__(self, rdc)
         self.user_resource_storage_path = os.path.join(
             self.config.GRASS_RESOURCE_DIR, self.user_id
         )
+        self.olderthan = args[1]
 
     def _execute(self):
 
@@ -59,8 +61,22 @@ def _execute(self):
         if os.path.exists(self.user_resource_storage_path) and os.path.isdir(
             self.user_resource_storage_path
         ):
-            executable = "/bin/rm"
-            args = ["-rf", self.user_resource_storage_path]
+
+            if self.olderthan is None:
+                # delete all user resources
+                executable = "/bin/rm"
+                args = ["-rf", self.user_resource_storage_path]
+            else:
+                # delete all user resources older than X days
+                executable = "/usr/bin/find"
+                args = [
+                    self.user_resource_storage_path,
+                    "-mindepth",
+                    "1",
+                    "-mtime",
+                    f"+{self.olderthan}",
+                    "-delete",
+                ]
 
             self._run_process(
                 Process(
@@ -71,7 +87,8 @@ def _execute(self):
                 )
             )
 
-            os.mkdir(self.user_resource_storage_path)
+            if not os.path.exists(self.user_resource_storage_path):
+                os.mkdir(self.user_resource_storage_path)
             self.finish_message = "Resource storage successfully removed."
         else:
             raise AsyncProcessError(

src/actinia_core/rest/resource_storage_management.py

@@ -28,6 +28,7 @@
 """
 
 from flask import jsonify, make_response
+from flask_restful import reqparse
 from flask_restful_swagger_2 import swagger
 import pickle
 from actinia_api.swagger2.actinia_core.apidocs import (
@@ -51,7 +52,7 @@
 from actinia_core.rest.base.user_auth import check_user_permissions
 
 __license__ = "GPLv3"
-__author__ = "Sören Gebbert"
+__author__ = "Sören Gebbert, Anika Weinmann"
 __copyright__ = (
     "Copyright 2016-2022, Sören Gebbert and mundialis GmbH & Co. KG"
 )
@@ -86,6 +87,26 @@ def get(self):
 
         return make_response(jsonify(response_model), http_code)
 
+    def _create_parser(self):
+        """Create the delete option arguments
+
+        The parameter contain:
+
+            olderthan : for older than X days
+
+        Returns:
+            The argument parser
+
+        """
+        parser = reqparse.RequestParser()
+        parser.add_argument(
+            "olderthan",
+            type=int,
+            location="args",
+            help="Older than x days. X must be specified as integer value",
+        )
+        return parser
+
     @endpoint_decorator()
     @swagger.doc(
         check_endpoint("delete", resource_storage_management.delete_doc)
@@ -94,11 +115,18 @@ def delete(self):
         """Clean the resource storage and remove all cached data"""
         rdc = self.preprocess(has_json=False, has_xml=False)
 
+        olderthan = None
+        parser = self._create_parser()
+        args = parser.parse_args()
+        if "olderthan" in args and args["olderthan"] is not None:
+            olderthan = args["olderthan"]
+
         if rdc:
             enqueue_job(
                 self.job_timeout,
                 start_resource_storage_remove,
                 rdc,
+                olderthan,
                 queue_type_overwrite=True,
             )
             http_code, response_model = self.wait_until_finish()

tests/test_resource_storage.py

@@ -25,6 +25,7 @@
 Tests: Resource storage test case
 """
 from flask.json import loads as json_load
+from datetime import datetime, timedelta
 import unittest
 import os
 
@@ -42,12 +43,11 @@
     )
 
 __license__ = "GPLv3"
-__author__ = "Sören Gebbert"
+__author__ = "Sören Gebbert, Anika Weinmann"
 __copyright__ = (
-    "Copyright 2016-2018, Sören Gebbert and mundialis GmbH & Co. KG"
+    "Copyright 2016-2022, Sören Gebbert and mundialis GmbH & Co. KG"
 )
-__maintainer__ = "Sören Gebbert"
-__email__ = "[email protected]"
+__maintainer__ = "mundialis"
 
 
 class ResourceStorageTestCase(ActiniaResourceTestCaseBase):
@@ -70,7 +70,6 @@ def test_resource_storage(self):
         rv = self.server.get(
             URL_PREFIX + "/resource_storage", headers=self.admin_auth_header
         )
-        print(rv.data)
         self.assertEqual(
             rv.status_code,
             200,
@@ -90,7 +89,6 @@ def test_resource_storage(self):
         rv = self.server.delete(
             URL_PREFIX + "/resource_storage", headers=self.admin_auth_header
         )
-        print(rv.data)
         self.assertEqual(
             rv.status_code,
             200,
@@ -103,7 +101,6 @@ def test_resource_storage(self):
         rv = self.server.get(
             URL_PREFIX + "/resource_storage", headers=self.admin_auth_header
         )
-        print(rv.data)
         self.assertEqual(
             rv.status_code,
             200,
@@ -128,7 +125,6 @@ def test_resource_storage_error_1(self):
         rv = self.server.get(
             URL_PREFIX + "/resource_storage", headers=self.admin_auth_header
         )
-        print(rv.data)
         self.assertEqual(
             rv.status_code,
             400,
@@ -146,7 +142,6 @@ def test_resource_storage_error_2(self):
         rv = self.server.delete(
             URL_PREFIX + "/resource_storage", headers=self.admin_auth_header
         )
-        print(rv.data)
         self.assertEqual(
             rv.status_code,
             400,
@@ -156,6 +151,82 @@ def test_resource_storage_error_2(self):
             rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype
         )
 
+    def test_resource_storage_delete_olderthan(self):
+
+        global_config.GRASS_RESOURCE_DIR = "/tmp/rstorage_tmp"
+        global_config.GRASS_RESOURCE_QUOTA = 1
+        try:
+            os.mkdir(global_config.GRASS_RESOURCE_DIR)
+        except Exception:  # more precise exception gladly accepted
+            pass
+
+        admin_resource_path = os.path.join(
+            global_config.GRASS_RESOURCE_DIR,
+            self.admin_id,
+        )
+        try:
+            os.mkdir(admin_resource_path)
+        except Exception:  # more precise exception gladly accepted
+            pass
+
+        # create files from specified date
+        now = datetime.now()
+        before_15days = now - timedelta(days=15)
+        file1 = os.path.join(admin_resource_path, "file1.txt")
+        file2 = os.path.join(admin_resource_path, "file2.txt")
+        date1 = now.strftime("%Y-%m-%d %H:%M:%S")
+        date2 = before_15days.strftime("%Y-%m-%d %H:%M:%S")
+        cmd_touch_file1 = f"touch -d '{date1}' {file1}"
+        cmd_touch_file2 = f"touch -d '{date2}' {file2}"
+        os.system(cmd_touch_file1)
+        os.system(cmd_touch_file2)
+        created_files = os.listdir(admin_resource_path)
+        self.assertIn(
+            "file1.txt",
+            created_files,
+            "'file1.txt' not in resource path after creation",
+        )
+        self.assertIn(
+            "file2.txt",
+            created_files,
+            "'file2.txt' not in resource path after creation",
+        )
+
+        # request resource storage
+        rv = self.server.get(
+            URL_PREFIX + "/resource_storage", headers=self.admin_auth_header
+        )
+        self.assertEqual(
+            rv.status_code,
+            200,
+            "HTML status code is wrong %i" % rv.status_code,
+        )
+
+        # delete files older than 10 days
+        rv = self.server.delete(
+            URL_PREFIX + "/resource_storage?olderthan=10",
+            headers=self.admin_auth_header,
+        )
+        self.assertEqual(
+            rv.status_code,
+            200,
+            "HTML status code is wrong %i" % rv.status_code,
+        )
+        # check files
+        files = os.listdir(admin_resource_path)
+        self.assertIn("file1.txt", files, "'file1.txt' not in resource path")
+        self.assertNotIn("file2.txt", files, "'file2.txt' in resource path")
+
+        # clean up resource storage
+        rv = self.server.delete(
+            URL_PREFIX + "/resource_storage", headers=self.admin_auth_header
+        )
+        self.assertEqual(
+            rv.status_code,
+            200,
+            "HTML status code is wrong %i" % rv.status_code,
+        )
+
 
 if __name__ == "__main__":
     unittest.main()