New webdav_server fixture

datalad_next/commands/tests/test_create_sibling_webdav.py

@@ -12,67 +12,65 @@
     assert_raises,
     assert_result_count,
     assert_status,
+    create_tree,
     eq_,
     ok_,
     run_main,
-    serve_path_via_webdav,
-    with_tempfile,
-    with_tree
 )
 import pytest
 
 from datalad.api import (
     clone,
     create_sibling_webdav,
 )
-from datalad_next.datasets import Dataset
 from datalad_next.utils import chpwd
 
 
-webdav_cred = ('dltest-my&=webdav', 'datalad', 'secure')
+def test_common_workflow_implicit_cred(
+        credman, existing_dataset, tmp_path, webdav_credential, webdav_server):
+    check_common_workflow(
+        False, 'annex',
+        credman, existing_dataset, tmp_path, webdav_credential, webdav_server)
 
 
-def test_common_workflow_implicit_cred(credman):
-    check_common_workflow(credman, False, 'annex')
+def test_common_workflow_explicit_cred(
+        credman, existing_dataset, tmp_path, webdav_credential, webdav_server):
+    check_common_workflow(
+        True, 'annex',
+        credman, existing_dataset, tmp_path, webdav_credential, webdav_server)
 
 
-def test_common_workflow_explicit_cred(credman):
-    check_common_workflow(credman, True, 'annex')
+def test_common_workflow_export(
+        credman, existing_dataset, tmp_path, webdav_credential, webdav_server):
+    check_common_workflow(
+        False, 'filetree',
+        credman, existing_dataset, tmp_path, webdav_credential, webdav_server)
 
 
-def test_common_workflow_export(credman):
-    check_common_workflow(credman, False, 'filetree')
-
-
-@with_tempfile
-@with_tempfile
-@with_tempfile
-@serve_path_via_webdav(auth=webdav_cred[1:])
 def check_common_workflow(
-        credman, declare_credential, mode,
-        clonepath, localpath, remotepath, url):
-    credman.set(webdav_cred[0], user=webdav_cred[1], secret=webdav_cred[2],
-                type='user_password')
+        declare_credential, mode,
+        credman, ds, clonepath, webdav_credential, webdav_server):
+    credman.set(**webdav_credential)
     ca = dict(result_renderer='disabled')
-    ds = Dataset(localpath).create(**ca)
     # need to amend the test credential, can only do after we know the URL
     ds.credentials(
         'set',
-        name=webdav_cred[0],
+        name=webdav_credential['name'],
         # the test webdav webserver uses a realm label '/'
-        spec=dict(realm=url + '/'),
+        spec=dict(realm=webdav_server.url + '/'),
         **ca)
 
     # we use a nasty target directory that has the potential to ruin the
     # git-remote URL handling
     targetdir_name = 'tar&get=mike'
-    targetdir = Path(remotepath) / targetdir_name
-    url = f'{url}/{targetdir_name}'
+    targetdir = Path(webdav_server.path) / targetdir_name
+    url = f'{webdav_server.url}/{targetdir_name}'
 
     with chpwd(ds.path):
         res = create_sibling_webdav(
             url,
-            credential=webdav_cred[0] if declare_credential else None,
+            credential=webdav_credential['name']
+            if declare_credential else None,
             mode=mode,
             **ca)
     assert_in_results(
@@ -96,7 +94,7 @@ def check_common_workflow(
         exp='yes' if 'filetree' in mode else 'no',
     )
     if declare_credential:
-        dlaurl += f'&dlacredential={urlquote(webdav_cred[0])}'
+        dlaurl += f'&dlacredential={urlquote(webdav_credential["name"])}'
 
     assert_in_results(
         res,
@@ -240,32 +238,33 @@ def test_unused_storage_name_warning(existing_dataset):
         eq_(lgr_mock.warning.call_count, len(mode_values))
 
 
-def test_existing_switch(credman):
-    credman.set('dltest-mywebdav', user=webdav_cred[1], secret=webdav_cred[2],
-                type='user_password')
-    check_existing_switch()
+def test_existing_switch(existing_dataset, credman, webdav_credential,
+                         webdav_server):
+    credman.set(**webdav_credential)
+    check_existing_switch(existing_dataset, webdav_credential, webdav_server)
 
 
-@with_tree(tree={'sub': {'f0': '0'},
-                 'sub2': {'subsub': {'f1': '1'},
-                          'f2': '2'},
-                 'f3': '3'})
-@with_tempfile
-@serve_path_via_webdav(auth=webdav_cred[1:])
-def check_existing_switch(localpath=None, remotepath=None, url=None):
+def check_existing_switch(ds, webdav_credential, webdav_server):
     ca = dict(result_renderer='disabled')
-    ds = Dataset(localpath).create(force=True, **ca)
+    create_tree(
+        ds.path,
+        {'sub': {'f0': '0'},
+         'sub2': {'subsub': {'f1': '1'},
+                  'f2': '2'},
+         'f3': '3'}
+    )
     # use a tricky name: '3f7' will be the hashdir of the XDLRA
     # key containing the superdataset's datalad-annex archive after a push
     sub = ds.create('3f7', force=True, **ca)
     sub2 = ds.create('sub2', force=True, **ca)
     subsub = sub2.create('subsub', force=True, **ca)
     ds.save(recursive=True, **ca)
 
+    url = webdav_server.url
     # need to amend the test credential, can only do after we know the URL
     ds.credentials(
         'set',
-        name='dltest-mywebdav',
+        name=webdav_credential['name'],
         # the test webdav webserver uses a realm label '/'
         spec=dict(realm=url + '/'),
         **ca)
@@ -364,7 +363,7 @@ def check_existing_switch(localpath=None, remotepath=None, url=None):
     assert all("is already configured" in r['message'][0] for r in res)
     assert all(r['action'].startswith('create_sibling_webdav') for r in res)
 
-    srv_rt = Path(remotepath)
+    srv_rt = Path(webdav_server.path)
     (srv_rt / '3f7').rmdir()
     (srv_rt / 'sub2' / 'subsub').rmdir()
     (srv_rt / 'sub2').rmdir()
@@ -393,27 +392,17 @@ def check_existing_switch(localpath=None, remotepath=None, url=None):
     assert_in(new_root / 'sub2' / 'subsub', remote_content)
 
 
-def test_result_renderer(credman):
-    credman.set(webdav_cred[0], user=webdav_cred[1], secret=webdav_cred[2],
-                type='user_password')
-    check_result_renderer()
-
-
-@with_tempfile
-@with_tempfile
-@serve_path_via_webdav(auth=webdav_cred[1:])
-def check_result_renderer(localpath=None, remotepath=None, url=None):
-    ca = dict(result_renderer='disabled')
-    ds = Dataset(localpath).create(**ca)
+def test_result_renderer(existing_dataset, credman,
+                         webdav_credential, webdav_server):
     # need to amend the test credential, can only do after we know the URL
-    ds.credentials(
-        'set',
-        name=webdav_cred[0],
-        # the test webdav webserver uses a realm label '/'
-        spec=dict(realm=url + '/'),
-        **ca)
-
-    out, err = run_main(['create-sibling-webdav', '-d', localpath, url])
+    # the test webdav webserver uses a realm label '/'
+    credman.set(realm=f'{webdav_server.url}/', **webdav_credential)
+    # consume stdout to make test self-contained
+    out, err = run_main([
+        'create-sibling-webdav',
+        '-d', existing_dataset.path,
+        webdav_server.url,
+    ])
     # url is somehow reported
     assert_in('datalad-annex::?type=webdav', out)
     # and the two custom result renderings

datalad_next/conftest.py

@@ -16,4 +16,10 @@
     dataset,
     #function-scope, Dataset instance with underlying repository
     existing_dataset,
+    #function-scope, Dataset instance with underlying Git-only repository
+    existing_noannex_dataset,
+    # session-scope, standard webdav credential (full dict)
+    webdav_credential,
+    # function-scope, serve a local temp-path via WebDAV
+    webdav_server,
 )

datalad_next/credman/tests/test_credman.py

@@ -286,7 +286,7 @@ def test_credman_obtain(tmp_keyring, datalad_cfg):
 """
 
 
-def test_legacy_credentials(tmp_keyring, tmp_path, existing_dataset):
+def test_legacy_credentials(tmp_keyring, existing_dataset):
     # - the legacy code will only ever pick up a dataset credential, when
     #   PWD is inside a dataset
     # - we want all tests to bypass the actual system keyring
@@ -296,7 +296,7 @@ def test_legacy_credentials(tmp_keyring, tmp_path, existing_dataset):
     # - we need to make them one and the same thing, and the tmp_keyring
     #   fixture does this by replacing the keyring storage for the runtime
     #   of the test
-    with chpwd(tmp_path):
+    with chpwd(existing_dataset.path):
         check_legacy_credentials(tmp_keyring, existing_dataset)
 
 

datalad_next/gitremotes/tests/test_datalad_annex.py

@@ -13,6 +13,7 @@
 from pathlib import Path
 from stat import S_IREAD, S_IRGRP, S_IROTH
 from unittest.mock import patch
+from urllib.parse import quote as urlquote
 
 from datalad.api import (
     Dataset,
@@ -28,7 +29,6 @@
     rmtree,
     serve_path_via_http,
     with_tempfile,
-    serve_path_via_webdav,
 )
 from datalad_next.utils import on_windows
 from datalad_next.exceptions import CommandError
@@ -347,30 +347,25 @@ def test_submodule_url(servepath=None, url=None, workdir=None):
     eq_(tobesubds.id, subdsclone.id)
 
 
-def test_webdav_auth(credman):
-    credman.set('dltest-mystuff', user=webdav_cred[0], secret=webdav_cred[1],
-                type='user_password')
-    check_webdav_auth()
-
-
-@with_tempfile
-@with_tempfile
-@with_tempfile
-@serve_path_via_webdav(auth=webdav_cred)
-def check_webdav_auth(preppath=None, clnpath=None, remotepath=None, webdavurl=None):
+def test_webdav_auth(existing_noannex_dataset,
+                     tmp_path,
+                     credman,
+                     webdav_credential,
+                     webdav_server):
+    credman.set(**webdav_credential)
     # this is the dataset we want to roundtrip through webdav
-    ds = Dataset(preppath).create(annex=False, result_renderer='disabled')
+    ds = existing_noannex_dataset
 
     remoteurl = \
-        f'datalad-annex::{webdavurl}' \
+        f'datalad-annex::{webdav_server.url}' \
         '?type=webdav&url={noquery}&encryption=none&' \
-        'dlacredential=dltest-mystuff'
+        f'dlacredential={urlquote(webdav_credential["name"])}'
 
     ds.repo.call_git(['remote', 'add', 'dla', remoteurl])
 
     # roundtrip
     ds.repo.call_git(['push', '-u', 'dla', DEFAULT_BRANCH])
-    cln = clone(remoteurl, clnpath)
+    cln = clone(remoteurl, tmp_path)
     # must give the same thing
     eq_(ds.repo.get_hexsha(DEFAULT_BRANCH),
         cln.repo.get_hexsha(DEFAULT_BRANCH))

datalad_next/tests/fixtures.py

@@ -8,6 +8,7 @@
 from datalad_next.datasets import Dataset
 from datalad_next.tests.utils import (
     SkipTest,
+    WebDAVPath,
     external_versions,
     get_git_config_global_fpath,
     md5sum,
@@ -181,13 +182,15 @@ def credman(datalad_cfg, tmp_keyring):
 
 
 @pytest.fixture(autouse=False, scope="function")
-def dataset(tmp_path):
+def dataset(datalad_cfg, tmp_path_factory):
     """Provides a ``Dataset`` instance for a not-yet-existing repository
 
     The instance points to an existing temporary path, but ``create()``
     has not been called on it yet.
     """
-    ds = Dataset(tmp_path)
+    # must use the factory to get a unique path even when a concrete
+    # test also uses `tmp_path`
+    ds = Dataset(tmp_path_factory.mktemp("dataset"))
     yield ds
 
 
@@ -200,3 +203,48 @@ def existing_dataset(dataset):
     """
     dataset.create(result_renderer='disabled')
     yield dataset
+
+
+@pytest.fixture(autouse=False, scope="function")
+def existing_noannex_dataset(dataset):
+    """just like ``existing_dataset``, but created with ``annex=False``
+    """
+    dataset.create(annex=False, result_renderer='disabled')
+    yield dataset
+
+
+@pytest.fixture(autouse=False, scope="session")
+def webdav_credential():
+    yield dict(
+        name='dltest-my&=webdav',
+        user='datalad',
+        secret='secure',
+        type='user_password',
+    )
+
+
+@pytest.fixture(autouse=False, scope="function")
+def webdav_server(tmp_path_factory, webdav_credential):
+    """Provides a WebDAV server, serving a temporary directory
+
+    The fixtures yields an instance of ``WebDAVPath``, provides the
+    following essential attributes:
+
+    - ``path``: ``Path`` instance of the served temporary directory
+    - ``url``: HTTP URL to access the WebDAV server
+
+    Server access requires HTTP Basic authentication with the credential
+    provided by the ``webdav_credential`` fixture.
+    """
+    auth = (webdav_credential['user'], webdav_credential['secret'])
+    # must use the factory to get a unique path even when a concrete
+    # test also uses `tmp_path`
+    path = tmp_path_factory.mktemp("webdav")
+    # this looks a little awkward, but is done to avoid a change in
+    # WebDAVPath. It would be better to have WebDAVPath directly
+    # set `.url` internally, but that would require adjusting
+    # the old `serve_path_via_webdav`
+    server = WebDAVPath(path, auth=auth)
+    with server as server_url:
+        server.url = server_url
+        yield server

datalad_next/tests/test_testutils.py

@@ -1,6 +1,25 @@
-from pathlib import Path
 from webdav3.client import Client as DAVClient
 
+
+def test_serve_webdav_fixture(webdav_credential, webdav_server):
+    webdav_cfg = dict(
+        webdav_hostname=webdav_server.url,
+        webdav_login=webdav_credential['user'],
+        webdav_password=webdav_credential['secret'],
+        webdav_root='/',
+    )
+    webdav = DAVClient(webdav_cfg)
+    # plain use should work without error
+    webdav.list()
+    (webdav_server.path / 'probe').touch()
+    assert 'probe' in webdav.list()
+
+
+#
+# anything below tests deprecated code
+#
+
+from pathlib import Path
 from datalad_next.tests.utils import (
     ok_,
     with_tempfile,
@@ -27,6 +46,8 @@ def test_serve_webdav(localpath=None, remotepath=None, url=None):
     ok_('probe' in webdav.list())
 
 
+# while technically possible, there is no practical application of an
+# auth-less WebDAV deployment
 @with_tempfile
 @with_tempfile
 @serve_path_via_webdav