From 64bfee815964fd83a61e6828d7c34936ad3beb06 Mon Sep 17 00:00:00 2001 From: Nissy0409 Date: Mon, 22 Jul 2024 21:45:50 +0900 Subject: [PATCH] Create a mode that does not generate intermediate files in SftpUpload. --- cliboa/adapter/sftp.py | 45 +++++++++++++++++------------------- cliboa/scenario/load/sftp.py | 5 ++++ docs/modules/sftp_upload.md | 33 +++++++++++++------------- 3 files changed, 43 insertions(+), 40 deletions(-) diff --git a/cliboa/adapter/sftp.py b/cliboa/adapter/sftp.py index 5042933b..70a2f1d8 100644 --- a/cliboa/adapter/sftp.py +++ b/cliboa/adapter/sftp.py @@ -81,7 +81,6 @@ def execute(self, obj): return self._execute(obj[0], obj[1]) def _execute(self, func, kwargs): - if not func: raise ValueError("Function must not be empty.") @@ -214,7 +213,7 @@ def get_specific_file(self, src, dest): """ return (get_specific_file_func, {"src": src, "dest": dest}) - def put_file(self, src, dest, endfile_suffix=None): + def put_file(self, src, dest, put_intermediation, endfile_suffix=None): """ Upload file to sftp server @@ -233,8 +232,17 @@ def put_file(self, src, dest, endfile_suffix=None): Raises: IOError: failed to upload + :param put_intermediation: """ - return (put_file_func, {"src": src, "dest": dest, "endfile_suffix": endfile_suffix}) + return ( + put_file_func, + { + "src": src, + "dest": dest, + "put_intermediation": put_intermediation, + "endfile_suffix": endfile_suffix, + }, + ) def file_exists_check(self, dir, pattern, ignore_empty_file=False): """ @@ -319,6 +327,8 @@ def put_file_func(**kwargs): '.' as prefix is added to the file name while uploading and then the prefix '.' will be removed (rename the file) when upload complete. + The function to add and remove the prefix can be disabled + by specifying "None" for "put_intermediation". """ dirname = os.path.dirname(kwargs["dest"]) @@ -328,26 +338,6 @@ def put_file_func(**kwargs): except FileNotFoundError: kwargs["sftp"].mkdir(dirname) - tmp_dest = os.path.join(dirname, "." + os.path.basename(kwargs["dest"])) - - _logger = logging.getLogger(__name__) - - if _logger.isEnabledFor(logging.DEBUG): - - def cb(sent, size): - _logger.debug("Transfer %s / %s" % (sent, size)) - - file_size = os.stat(kwargs["src"]).st_size - with open(kwargs["src"], "rb") as fl: - _logger.debug("Open src file") - with kwargs["sftp"].file(tmp_dest, "wb") as fr: - _logger.debug("Open dest file") - fr.set_pipelined(True) - _transfer_with_callback(reader=fl, writer=fr, file_size=file_size, callback=cb) - _logger.debug("End") - else: - kwargs["sftp"].put(kwargs["src"], tmp_dest) - # Same file name is removed in advance, if exists try: f = kwargs["sftp"].stat(kwargs["dest"]) @@ -356,7 +346,14 @@ def cb(sent, size): except FileNotFoundError: pass - kwargs["sftp"].rename(tmp_dest, kwargs["dest"]) + if kwargs["put_intermediation"] is None: + kwargs["sftp"].put(kwargs["src"], os.path.join(dirname, os.path.basename(kwargs["dest"]))) + else: + tmp_dest = os.path.join( + dirname, kwargs["put_intermediation"] + os.path.basename(kwargs["dest"]) + ) + kwargs["sftp"].put(kwargs["src"], tmp_dest) + kwargs["sftp"].rename(tmp_dest, kwargs["dest"]) endfile_suffix = kwargs["endfile_suffix"] if endfile_suffix is not None: diff --git a/cliboa/scenario/load/sftp.py b/cliboa/scenario/load/sftp.py index 3b41e5ce..ea1a9e24 100644 --- a/cliboa/scenario/load/sftp.py +++ b/cliboa/scenario/load/sftp.py @@ -33,6 +33,7 @@ def __init__(self): super().__init__() self._quit = False self._ignore_empty_file = False + self._put_intermediation = "." def quit(self, quit): self._quit = quit @@ -40,6 +41,9 @@ def quit(self, quit): def ignore_empty_file(self, ignore_empty_file): self._ignore_empty_file = ignore_empty_file + def put_intermediation(self, put_intermediation): + self._put_intermediation = put_intermediation + def execute(self, *args): # essential parameters check valid = EssentialParameters( @@ -61,6 +65,7 @@ def execute(self, *args): ).put_file( src=file, dest=os.path.join(self._dest_dir, os.path.basename(file)), + put_intermediation=self._put_intermediation, endfile_suffix=self._endfile_suffix, ) adaptor.execute(obj) diff --git a/docs/modules/sftp_upload.md b/docs/modules/sftp_upload.md index b8746f4a..fa1620b5 100644 --- a/docs/modules/sftp_upload.md +++ b/docs/modules/sftp_upload.md @@ -2,22 +2,23 @@ Upload a file via SFTP. # Parameters -|Parameters|Explanation|Required|Default|Remarks| -|----------|-----------|--------|-------|-------| -|host|Host name or IP address of a sftp server.|Yes|None|| -|user|User name for authentication|Yes|None|| -|password|Password for authentication|No|None|Either password or key is required| -|key|Path to key for authentication|No|None|| -|passphrase|Used for decrypting key|No|None|| -|port|Port number of a sftp server|No|22|| -|src_dir|Directory of source to upload|Yes|None|| -|src_pattern|File pattern of source to upload. Regexp is available.|Yes|None|| -|dest_dir|Destination directory to upload.|No|None| -|endfile_suffix|Places file with original file name + ".endfile_suffix" when upload completed.|No|None|| -|timeout|Timeout period of sftp connection. Unit is second.|No|30|| -|retry_count|retry count of sftp connection.|No|3|| -|quit|True or False flag for quitting cliboa process when source files do not exist.|No|False|| -|ignore_empty_file|If True, size zero files are not be uploaded|No|False|| +| Parameters | Explanation | Required | Default | Remarks | +|--------------------|--------------------------------------------------------------------------------|----------|---------|-------------------------------------------------------------------------| +| host | Host name or IP address of a sftp server. | Yes | None | | +| user | User name for authentication | Yes | None | | +| password | Password for authentication | No | None | Either password or key is required | +| key | Path to key for authentication | No | None | | +| passphrase | Used for decrypting key | No | None | | +| port | Port number of a sftp server | No | 22 | | +| src_dir | Directory of source to upload | Yes | None | | +| src_pattern | File pattern of source to upload. Regexp is available. | Yes | None | | +| dest_dir | Destination directory to upload. | No | None | | +| endfile_suffix | Places file with original file name + ".endfile_suffix" when upload completed. | No | None | | +| timeout | Timeout period of sftp connection. Unit is second. | No | 30 | | +| retry_count | retry count of sftp connection. | No | 3 | | +| quit | True or False flag for quitting cliboa process when source files do not exist. | No | False | | +| ignore_empty_file | If True, size zero files are not be uploaded | No | False | | +| put_intermediation | Specify prefix to be added to the intermediate file created when PUT file. | No | . | If you do not want to create an intermediate file, define it as "None". | # Examples ```