From 5359c6e6bd49a6f094601ca0f2d44f9703888ee8 Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 21 Nov 2023 16:34:43 +0100 Subject: [PATCH 1/4] Fix issues --- faassupervisor/storage/config.py | 2 +- faassupervisor/storage/providers/s3.py | 2 +- test/unit/faassupervisor/storage.py | 9 ++++++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/faassupervisor/storage/config.py b/faassupervisor/storage/config.py index b7ea23d..c83c610 100644 --- a/faassupervisor/storage/config.py +++ b/faassupervisor/storage/config.py @@ -234,7 +234,7 @@ def upload_output(self, output_dir_path): provider_type = StrUtils.get_storage_type(output['storage_provider']) provider_id = StrUtils.get_storage_id(output['storage_provider']) for file_path in output_files: - file_name = file_path.replace(f'{output_dir_path}/', '') + file_name = file_path.replace(f'{output_dir_path}/', '').strip() prefix_ok = False suffix_ok = False # Check prefixes diff --git a/faassupervisor/storage/providers/s3.py b/faassupervisor/storage/providers/s3.py index d4e8936..50b8560 100644 --- a/faassupervisor/storage/providers/s3.py +++ b/faassupervisor/storage/providers/s3.py @@ -15,7 +15,7 @@ related with the S3 storage provider. """ import boto3 -from requests.packages import urllib3 +import urllib3 from faassupervisor.logger import get_logger from faassupervisor.storage.providers import DefaultStorageProvider, \ get_bucket_name, get_file_key diff --git a/test/unit/faassupervisor/storage.py b/test/unit/faassupervisor/storage.py index e27b84e..7d42832 100644 --- a/test/unit/faassupervisor/storage.py +++ b/test/unit/faassupervisor/storage.py @@ -259,7 +259,8 @@ def test_upload_output(self, mock_minio, mock_s3, mock_get_files): '/tmp/test/result-file.txt', '/tmp/test/result-file.out', '/tmp/test/file2.txt', - '/tmp/test/file2.out' + '/tmp/test/file2.out', + '/tmp/test/\nfile3.out' ] mock_get_files.return_value = files StorageConfig().upload_output('/tmp/test') @@ -268,10 +269,12 @@ def test_upload_output(self, mock_minio, mock_s3, mock_get_files): call('/tmp/test/result-file.txt', 'result-file.txt', 'bucket')) - self.assertEqual(mock_s3.call_count, 6) - for i, f in enumerate(files): + self.assertEqual(mock_s3.call_count, 7) + for i, f in enumerate(files[:-1]): self.assertEqual(mock_s3.call_args_list[i], call(f, f.split('/')[3], 'bucket/folder')) + self.assertEqual(mock_s3.call_args_list[6], + call('/tmp/test/\nfile3.out', 'file3.out', 'bucket/folder')) # def test_upload_real_output(self): # with mock.patch.dict('os.environ', From b35d39e26834ebd83f7ba15fd953ae91ebce2393 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Wed, 22 Nov 2023 08:28:28 +0100 Subject: [PATCH 2/4] Remove space --- faassupervisor/storage/providers/s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/faassupervisor/storage/providers/s3.py b/faassupervisor/storage/providers/s3.py index 50b8560..cc2e7b7 100644 --- a/faassupervisor/storage/providers/s3.py +++ b/faassupervisor/storage/providers/s3.py @@ -21,7 +21,7 @@ get_bucket_name, get_file_key from faassupervisor.utils import SysUtils -urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) class S3(DefaultStorageProvider): """Class that manages downloads and uploads from S3.""" From 00cd03b0af9f24f04789496824c5d11eea4d382c Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Wed, 22 Nov 2023 09:16:10 +0100 Subject: [PATCH 3/4] remove initial / --- faassupervisor/storage/config.py | 2 +- test/unit/faassupervisor/storage.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/faassupervisor/storage/config.py b/faassupervisor/storage/config.py index c83c610..64ffe42 100644 --- a/faassupervisor/storage/config.py +++ b/faassupervisor/storage/config.py @@ -234,7 +234,7 @@ def upload_output(self, output_dir_path): provider_type = StrUtils.get_storage_type(output['storage_provider']) provider_id = StrUtils.get_storage_id(output['storage_provider']) for file_path in output_files: - file_name = file_path.replace(f'{output_dir_path}/', '').strip() + file_name = file_path.replace(f'{output_dir_path}/', '').strip().lstrip('/') prefix_ok = False suffix_ok = False # Check prefixes diff --git a/test/unit/faassupervisor/storage.py b/test/unit/faassupervisor/storage.py index 7d42832..9b7228d 100644 --- a/test/unit/faassupervisor/storage.py +++ b/test/unit/faassupervisor/storage.py @@ -260,7 +260,7 @@ def test_upload_output(self, mock_minio, mock_s3, mock_get_files): '/tmp/test/result-file.out', '/tmp/test/file2.txt', '/tmp/test/file2.out', - '/tmp/test/\nfile3.out' + '/tmp/test/\n/file3.out' ] mock_get_files.return_value = files StorageConfig().upload_output('/tmp/test') @@ -274,7 +274,7 @@ def test_upload_output(self, mock_minio, mock_s3, mock_get_files): self.assertEqual(mock_s3.call_args_list[i], call(f, f.split('/')[3], 'bucket/folder')) self.assertEqual(mock_s3.call_args_list[6], - call('/tmp/test/\nfile3.out', 'file3.out', 'bucket/folder')) + call('/tmp/test/\n/file3.out', 'file3.out', 'bucket/folder')) # def test_upload_real_output(self): # with mock.patch.dict('os.environ', From 4b1271c6ef398ae0b6c53fc622b6d4bbfe278af0 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Wed, 22 Nov 2023 09:20:09 +0100 Subject: [PATCH 4/4] Add comment --- faassupervisor/storage/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/faassupervisor/storage/config.py b/faassupervisor/storage/config.py index 64ffe42..8ad1c6d 100644 --- a/faassupervisor/storage/config.py +++ b/faassupervisor/storage/config.py @@ -234,6 +234,7 @@ def upload_output(self, output_dir_path): provider_type = StrUtils.get_storage_type(output['storage_provider']) provider_id = StrUtils.get_storage_id(output['storage_provider']) for file_path in output_files: + # Make sure the file name does not contain new lines or starting slashes file_name = file_path.replace(f'{output_dir_path}/', '').strip().lstrip('/') prefix_ok = False suffix_ok = False