diff --git a/src/actinia_core/processing/actinia_processing/persistent/resource_storage_management.py b/src/actinia_core/processing/actinia_processing/persistent/resource_storage_management.py index baf627bfc..15b4cd3f7 100644 --- a/src/actinia_core/processing/actinia_processing/persistent/resource_storage_management.py +++ b/src/actinia_core/processing/actinia_processing/persistent/resource_storage_management.py @@ -36,7 +36,7 @@ __license__ = "GPLv3" -__author__ = "Sören Gebbert" +__author__ = "Sören Gebbert, Anika Weinmann" __copyright__ = ( "Copyright 2016-2022, Sören Gebbert and mundialis GmbH & Co. KG" ) @@ -47,10 +47,12 @@ class ResourceStorageDelete(PersistentProcessing): """Delete the user specific resource directory""" def __init__(self, *args): - PersistentProcessing.__init__(self, *args) + rdc = args[0] + PersistentProcessing.__init__(self, rdc) self.user_resource_storage_path = os.path.join( self.config.GRASS_RESOURCE_DIR, self.user_id ) + self.olderthan = args[1] def _execute(self): @@ -59,8 +61,22 @@ def _execute(self): if os.path.exists(self.user_resource_storage_path) and os.path.isdir( self.user_resource_storage_path ): - executable = "/bin/rm" - args = ["-rf", self.user_resource_storage_path] + + if self.olderthan is None: + # delete all user resources + executable = "/bin/rm" + args = ["-rf", self.user_resource_storage_path] + else: + # delete all user resources older than X days + executable = "/usr/bin/find" + args = [ + self.user_resource_storage_path, + "-mindepth", + "1", + "-mtime", + f"+{self.olderthan}", + "-delete", + ] self._run_process( Process( @@ -71,7 +87,8 @@ def _execute(self): ) ) - os.mkdir(self.user_resource_storage_path) + if not os.path.exists(self.user_resource_storage_path): + os.mkdir(self.user_resource_storage_path) self.finish_message = "Resource storage successfully removed." else: raise AsyncProcessError( diff --git a/src/actinia_core/rest/resource_storage_management.py b/src/actinia_core/rest/resource_storage_management.py index de3a00d80..4438d44aa 100644 --- a/src/actinia_core/rest/resource_storage_management.py +++ b/src/actinia_core/rest/resource_storage_management.py @@ -28,6 +28,7 @@ """ from flask import jsonify, make_response +from flask_restful import reqparse from flask_restful_swagger_2 import swagger import pickle from actinia_api.swagger2.actinia_core.apidocs import ( @@ -51,7 +52,7 @@ from actinia_core.rest.base.user_auth import check_user_permissions __license__ = "GPLv3" -__author__ = "Sören Gebbert" +__author__ = "Sören Gebbert, Anika Weinmann" __copyright__ = ( "Copyright 2016-2022, Sören Gebbert and mundialis GmbH & Co. KG" ) @@ -86,6 +87,26 @@ def get(self): return make_response(jsonify(response_model), http_code) + def _create_parser(self): + """Create the delete option arguments + + The parameter contain: + + olderthan : for older than X days + + Returns: + The argument parser + + """ + parser = reqparse.RequestParser() + parser.add_argument( + "olderthan", + type=int, + location="args", + help="Older than x days. X must be specified as integer value", + ) + return parser + @endpoint_decorator() @swagger.doc( check_endpoint("delete", resource_storage_management.delete_doc) @@ -94,11 +115,18 @@ def delete(self): """Clean the resource storage and remove all cached data""" rdc = self.preprocess(has_json=False, has_xml=False) + olderthan = None + parser = self._create_parser() + args = parser.parse_args() + if "olderthan" in args and args["olderthan"] is not None: + olderthan = args["olderthan"] + if rdc: enqueue_job( self.job_timeout, start_resource_storage_remove, rdc, + olderthan, queue_type_overwrite=True, ) http_code, response_model = self.wait_until_finish() diff --git a/tests/test_resource_storage.py b/tests/test_resource_storage.py index f1939319d..82aa6ad42 100644 --- a/tests/test_resource_storage.py +++ b/tests/test_resource_storage.py @@ -25,6 +25,7 @@ Tests: Resource storage test case """ from flask.json import loads as json_load +from datetime import datetime, timedelta import unittest import os @@ -42,12 +43,11 @@ ) __license__ = "GPLv3" -__author__ = "Sören Gebbert" +__author__ = "Sören Gebbert, Anika Weinmann" __copyright__ = ( - "Copyright 2016-2018, Sören Gebbert and mundialis GmbH & Co. KG" + "Copyright 2016-2022, Sören Gebbert and mundialis GmbH & Co. KG" ) -__maintainer__ = "Sören Gebbert" -__email__ = "soerengebbert@googlemail.com" +__maintainer__ = "mundialis" class ResourceStorageTestCase(ActiniaResourceTestCaseBase): @@ -70,7 +70,6 @@ def test_resource_storage(self): rv = self.server.get( URL_PREFIX + "/resource_storage", headers=self.admin_auth_header ) - print(rv.data) self.assertEqual( rv.status_code, 200, @@ -90,7 +89,6 @@ def test_resource_storage(self): rv = self.server.delete( URL_PREFIX + "/resource_storage", headers=self.admin_auth_header ) - print(rv.data) self.assertEqual( rv.status_code, 200, @@ -103,7 +101,6 @@ def test_resource_storage(self): rv = self.server.get( URL_PREFIX + "/resource_storage", headers=self.admin_auth_header ) - print(rv.data) self.assertEqual( rv.status_code, 200, @@ -128,7 +125,6 @@ def test_resource_storage_error_1(self): rv = self.server.get( URL_PREFIX + "/resource_storage", headers=self.admin_auth_header ) - print(rv.data) self.assertEqual( rv.status_code, 400, @@ -146,7 +142,6 @@ def test_resource_storage_error_2(self): rv = self.server.delete( URL_PREFIX + "/resource_storage", headers=self.admin_auth_header ) - print(rv.data) self.assertEqual( rv.status_code, 400, @@ -156,6 +151,82 @@ def test_resource_storage_error_2(self): rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype ) + def test_resource_storage_delete_olderthan(self): + + global_config.GRASS_RESOURCE_DIR = "/tmp/rstorage_tmp" + global_config.GRASS_RESOURCE_QUOTA = 1 + try: + os.mkdir(global_config.GRASS_RESOURCE_DIR) + except Exception: # more precise exception gladly accepted + pass + + admin_resource_path = os.path.join( + global_config.GRASS_RESOURCE_DIR, + self.admin_id, + ) + try: + os.mkdir(admin_resource_path) + except Exception: # more precise exception gladly accepted + pass + + # create files from specified date + now = datetime.now() + before_15days = now - timedelta(days=15) + file1 = os.path.join(admin_resource_path, "file1.txt") + file2 = os.path.join(admin_resource_path, "file2.txt") + date1 = now.strftime("%Y-%m-%d %H:%M:%S") + date2 = before_15days.strftime("%Y-%m-%d %H:%M:%S") + cmd_touch_file1 = f"touch -d '{date1}' {file1}" + cmd_touch_file2 = f"touch -d '{date2}' {file2}" + os.system(cmd_touch_file1) + os.system(cmd_touch_file2) + created_files = os.listdir(admin_resource_path) + self.assertIn( + "file1.txt", + created_files, + "'file1.txt' not in resource path after creation", + ) + self.assertIn( + "file2.txt", + created_files, + "'file2.txt' not in resource path after creation", + ) + + # request resource storage + rv = self.server.get( + URL_PREFIX + "/resource_storage", headers=self.admin_auth_header + ) + self.assertEqual( + rv.status_code, + 200, + "HTML status code is wrong %i" % rv.status_code, + ) + + # delete files older than 10 days + rv = self.server.delete( + URL_PREFIX + "/resource_storage?olderthan=10", + headers=self.admin_auth_header, + ) + self.assertEqual( + rv.status_code, + 200, + "HTML status code is wrong %i" % rv.status_code, + ) + # check files + files = os.listdir(admin_resource_path) + self.assertIn("file1.txt", files, "'file1.txt' not in resource path") + self.assertNotIn("file2.txt", files, "'file2.txt' in resource path") + + # clean up resource storage + rv = self.server.delete( + URL_PREFIX + "/resource_storage", headers=self.admin_auth_header + ) + self.assertEqual( + rv.status_code, + 200, + "HTML status code is wrong %i" % rv.status_code, + ) + if __name__ == "__main__": unittest.main()