Skip to content

Commit

Permalink
feat(atomic): describe atomicity property
Browse files Browse the repository at this point in the history
This commit adds a description of the atomicity
properties of upload to the doc-string of the method
`url_operations.base.UrlOperations.upload`.
  • Loading branch information
christian-monch committed Jul 1, 2024
1 parent 4a8aa14 commit f07a613
Show file tree
Hide file tree
Showing 3 changed files with 12 additions and 9 deletions.
5 changes: 5 additions & 0 deletions datalad_next/url_operations/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,11 @@ def upload(self,
timeout: float | None = None) -> Dict:
"""Upload from a local file or stream to a URL
Whenever possible, uploads are performed atomically This means that the
destination will never see a partially uploaded file. It will either
see the previous content (or nothing) or the newly uploaded content.
Note: this is not supported by all implementations of URL-operations.
Parameters
----------
from_path: Path or None
Expand Down
9 changes: 4 additions & 5 deletions datalad_next/url_operations/tests/test_file.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,7 @@ def test_file_url_download(tmp_path):
ops.download(test_url, download_path)


@pytest.mark.parametrize('atomic', [True, False])
def test_file_url_upload(tmp_path, monkeypatch, atomic):
def test_file_url_upload(tmp_path, monkeypatch):
payload = 'payload'
payload_file = tmp_path / 'payload'
test_upload_path = tmp_path / 'myfile'
Expand All @@ -48,13 +47,13 @@ def test_file_url_upload(tmp_path, monkeypatch, atomic):
# missing source file
# standard exception, makes no sense to go custom thinks mih
with pytest.raises(FileNotFoundError):
ops.upload(payload_file, test_upload_url, atomic=atomic)
ops.upload(payload_file, test_upload_url)
# no empty targets lying around
assert not test_upload_path.exists()

# now again
payload_file.write_text(payload)
props = ops.upload(payload_file, test_upload_url, hash=['md5'], atomic=atomic)
props = ops.upload(payload_file, test_upload_url, hash=['md5'])
assert test_upload_path.read_text() == 'payload'
assert props['content-length'] == len(payload)
assert props['md5'] == '321c3cf486ed509164edec1e1981fec8'
Expand All @@ -65,7 +64,7 @@ def test_file_url_upload(tmp_path, monkeypatch, atomic):
m.setattr(sys, 'stdin',
io.TextIOWrapper(io.BytesIO(
bytes(payload, encoding='utf-8'))))
props = ops.upload(None, from_stdin_url, hash=['md5'], atomic=atomic)
props = ops.upload(None, from_stdin_url, hash=['md5'])
assert props['md5'] == '321c3cf486ed509164edec1e1981fec8'
assert props['content-length'] == len(payload)

Expand Down
7 changes: 3 additions & 4 deletions datalad_next/url_operations/tests/test_ssh.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,7 @@ def test_ssh_url_download(tmp_path, monkeypatch, sshserver):

# path magic inside the test is posix only
@skip_if_on_windows
@pytest.mark.parametrize('atomic', [True, False])
def test_ssh_url_upload(tmp_path, monkeypatch, sshserver, atomic):
def test_ssh_url_upload(tmp_path, monkeypatch, sshserver):
ssh_url, ssh_localpath = sshserver
payload = 'surprise!'
payload_path = tmp_path / 'payload'
Expand All @@ -60,7 +59,7 @@ def test_ssh_url_upload(tmp_path, monkeypatch, sshserver, atomic):

# standard error if local source is not around
with pytest.raises(FileNotFoundError):
ops.upload(payload_path, upload_url, atomic=atomic)
ops.upload(payload_path, upload_url)

payload_path.write_text(payload)
# upload creates parent dirs, so the next just works.
Expand All @@ -69,7 +68,7 @@ def test_ssh_url_upload(tmp_path, monkeypatch, sshserver, atomic):
# server-side preconditions first.
# this functionality is not about exposing a full
# remote FS abstraction -- just upload
ops.upload(payload_path, upload_url, atomic=atomic)
ops.upload(payload_path, upload_url)
assert upload_path.read_text() == payload


Expand Down

0 comments on commit f07a613

Please sign in to comment.