Skip to content

Commit

Permalink
Create a mode that does not generate intermediate files in SftpUpload.
Browse files Browse the repository at this point in the history
  • Loading branch information
@nissy0409240
nissy0409240 committed Jul 22, 2024
1 parent 0d94f86 commit 64bfee8
Show file tree
Hide file tree
Showing 3 changed files with 43 additions and 40 deletions.
45 changes: 21 additions & 24 deletions cliboa/adapter/sftp.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,6 @@ def execute(self, obj):
return self._execute(obj[0], obj[1])

def _execute(self, func, kwargs):

if not func:
raise ValueError("Function must not be empty.")

Expand Down Expand Up @@ -214,7 +213,7 @@ def get_specific_file(self, src, dest):
"""
return (get_specific_file_func, {"src": src, "dest": dest})

def put_file(self, src, dest, endfile_suffix=None):
def put_file(self, src, dest, put_intermediation, endfile_suffix=None):
"""
Upload file to sftp server
Expand All @@ -233,8 +232,17 @@ def put_file(self, src, dest, endfile_suffix=None):
Raises:
IOError: failed to upload
:param put_intermediation:
"""
return (put_file_func, {"src": src, "dest": dest, "endfile_suffix": endfile_suffix})
return (
put_file_func,
{
"src": src,
"dest": dest,
"put_intermediation": put_intermediation,
"endfile_suffix": endfile_suffix,
},
)

def file_exists_check(self, dir, pattern, ignore_empty_file=False):
"""
Expand Down Expand Up @@ -319,6 +327,8 @@ def put_file_func(**kwargs):
'.' as prefix is added to the file name
while uploading and then the prefix '.'
will be removed (rename the file) when upload complete.
The function to add and remove the prefix can be disabled
by specifying "None" for "put_intermediation".
"""
dirname = os.path.dirname(kwargs["dest"])

Expand All @@ -328,26 +338,6 @@ def put_file_func(**kwargs):
except FileNotFoundError:
kwargs["sftp"].mkdir(dirname)

tmp_dest = os.path.join(dirname, "." + os.path.basename(kwargs["dest"]))

_logger = logging.getLogger(__name__)

if _logger.isEnabledFor(logging.DEBUG):

def cb(sent, size):
_logger.debug("Transfer %s / %s" % (sent, size))

file_size = os.stat(kwargs["src"]).st_size
with open(kwargs["src"], "rb") as fl:
_logger.debug("Open src file")
with kwargs["sftp"].file(tmp_dest, "wb") as fr:
_logger.debug("Open dest file")
fr.set_pipelined(True)
_transfer_with_callback(reader=fl, writer=fr, file_size=file_size, callback=cb)
_logger.debug("End")
else:
kwargs["sftp"].put(kwargs["src"], tmp_dest)

# Same file name is removed in advance, if exists
try:
f = kwargs["sftp"].stat(kwargs["dest"])
Expand All @@ -356,7 +346,14 @@ def cb(sent, size):
except FileNotFoundError:
pass

kwargs["sftp"].rename(tmp_dest, kwargs["dest"])
if kwargs["put_intermediation"] is None:
kwargs["sftp"].put(kwargs["src"], os.path.join(dirname, os.path.basename(kwargs["dest"])))
else:
tmp_dest = os.path.join(
dirname, kwargs["put_intermediation"] + os.path.basename(kwargs["dest"])
)
kwargs["sftp"].put(kwargs["src"], tmp_dest)
kwargs["sftp"].rename(tmp_dest, kwargs["dest"])

endfile_suffix = kwargs["endfile_suffix"]
if endfile_suffix is not None:
Expand Down
5 changes: 5 additions & 0 deletions cliboa/scenario/load/sftp.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,17 @@ def __init__(self):
super().__init__()
self._quit = False
self._ignore_empty_file = False
self._put_intermediation = "."

def quit(self, quit):
self._quit = quit

def ignore_empty_file(self, ignore_empty_file):
self._ignore_empty_file = ignore_empty_file

def put_intermediation(self, put_intermediation):
self._put_intermediation = put_intermediation

def execute(self, *args):
# essential parameters check
valid = EssentialParameters(
Expand All @@ -61,6 +65,7 @@ def execute(self, *args):
).put_file(
src=file,
dest=os.path.join(self._dest_dir, os.path.basename(file)),
put_intermediation=self._put_intermediation,
endfile_suffix=self._endfile_suffix,
)
adaptor.execute(obj)
Expand Down
33 changes: 17 additions & 16 deletions docs/modules/sftp_upload.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,22 +2,23 @@
Upload a file via SFTP.

# Parameters
|Parameters|Explanation|Required|Default|Remarks|
|----------|-----------|--------|-------|-------|
|host|Host name or IP address of a sftp server.|Yes|None||
|user|User name for authentication|Yes|None||
|password|Password for authentication|No|None|Either password or key is required|
|key|Path to key for authentication|No|None||
|passphrase|Used for decrypting key|No|None||
|port|Port number of a sftp server|No|22||
|src_dir|Directory of source to upload|Yes|None||
|src_pattern|File pattern of source to upload. Regexp is available.|Yes|None||
|dest_dir|Destination directory to upload.|No|None|
|endfile_suffix|Places file with original file name + ".endfile_suffix" when upload completed.|No|None||
|timeout|Timeout period of sftp connection. Unit is second.|No|30||
|retry_count|retry count of sftp connection.|No|3||
|quit|True or False flag for quitting cliboa process when source files do not exist.|No|False||
|ignore_empty_file|If True, size zero files are not be uploaded|No|False||
| Parameters | Explanation | Required | Default | Remarks |
|--------------------|--------------------------------------------------------------------------------|----------|---------|-------------------------------------------------------------------------|
| host | Host name or IP address of a sftp server. | Yes | None | |
| user | User name for authentication | Yes | None | |
| password | Password for authentication | No | None | Either password or key is required |
| key | Path to key for authentication | No | None | |
| passphrase | Used for decrypting key | No | None | |
| port | Port number of a sftp server | No | 22 | |
| src_dir | Directory of source to upload | Yes | None | |
| src_pattern | File pattern of source to upload. Regexp is available. | Yes | None | |
| dest_dir | Destination directory to upload. | No | None | |
| endfile_suffix | Places file with original file name + ".endfile_suffix" when upload completed. | No | None | |
| timeout | Timeout period of sftp connection. Unit is second. | No | 30 | |
| retry_count | retry count of sftp connection. | No | 3 | |
| quit | True or False flag for quitting cliboa process when source files do not exist. | No | False | |
| ignore_empty_file | If True, size zero files are not be uploaded | No | False | |
| put_intermediation | Specify prefix to be added to the intermediate file created when PUT file. | No | . | If you do not want to create an intermediate file, define it as "None". |

# Examples
```
Expand Down

0 comments on commit 64bfee8

Please sign in to comment.