remote: add support for WebDAV

dvc/config.py

@@ -184,6 +184,8 @@ class RelPath(str):
                 },
                 "http": {**HTTP_COMMON, **REMOTE_COMMON},
                 "https": {**HTTP_COMMON, **REMOTE_COMMON},
+                "webdav": {**HTTP_COMMON, **REMOTE_COMMON},
+                "webdavs": {**HTTP_COMMON, **REMOTE_COMMON},
                 "remote": {str: object},  # Any of the above options are valid
             }
         )

dvc/path_info.py

@@ -312,3 +312,33 @@ def __eq__(self, other):
             and self._path == other._path
             and self._extra_parts == other._extra_parts
         )
+
+
+class WebdavURLInfo(HTTPURLInfo):
+    def __init__(self, url):
+        super().__init__(url)
+
+    @cached_property
+    def url(self):
+        return "{}://{}{}{}{}{}".format(
+            self.scheme.replace("webdav", "http"),
+            self.netloc,
+            self._spath,
+            (";" + self.params) if self.params else "",
+            ("?" + self.query) if self.query else "",
+            ("#" + self.fragment) if self.fragment else "",
+        )
+
+    def get_collections(self) -> list:
+        def pcol(path):
+            return "{}://{}{}".format(
+                self.scheme.replace("webdav", "http"), self.netloc, path,
+            )
+
+        p = self.path.split("/")[1:-1]
+        if not p:
+            return []
+        r = []
+        for i in range(len(p)):
+            r.append(pcol("/{}/".format("/".join(p[: i + 1]))))
+        return r

dvc/remote/__init__.py

@@ -11,6 +11,8 @@
 from dvc.remote.oss import RemoteOSS
 from dvc.remote.s3 import RemoteS3
 from dvc.remote.ssh import RemoteSSH
+from dvc.remote.webdav import RemoteWEBDAV
+from dvc.remote.webdavs import RemoteWEBDAVS
 
 
 REMOTES = [
@@ -23,6 +25,8 @@
     RemoteS3,
     RemoteSSH,
     RemoteOSS,
+    RemoteWEBDAV,
+    RemoteWEBDAVS,
     # NOTE: RemoteLOCAL is the default
 ]
 

dvc/remote/webdav.py

@@ -0,0 +1,60 @@
+import os.path
+
+from .http import RemoteHTTP
+from dvc.scheme import Schemes
+from dvc.progress import Tqdm
+from dvc.exceptions import HTTPError
+from dvc.path_info import WebdavURLInfo
+
+
+class RemoteWEBDAV(RemoteHTTP):
+    scheme = Schemes.WEBDAV
+    path_cls = WebdavURLInfo
+    REQUEST_TIMEOUT = 20
+
+    def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
+        def chunks():
+            with open(from_file, "rb") as fd:
+                with Tqdm.wrapattr(
+                    fd,
+                    "read",
+                    total=None
+                    if no_progress_bar
+                    else os.path.getsize(from_file),
+                    leave=False,
+                    desc=to_info.url if name is None else name,
+                    disable=no_progress_bar,
+                ) as fd_wrapped:
+                    while True:
+                        chunk = fd_wrapped.read(self.CHUNK_SIZE)
+                        if not chunk:
+                            break
+                        yield chunk
+
+        self._create_collections(to_info)
+        response = self._request("PUT", to_info.url, data=chunks())
+        if response.status_code not in (200, 201):
+            raise HTTPError(response.status_code, response.reason)
+
+    def _create_collections(self, to_info):
+        url_cols = to_info.get_collections()
+        from_idx = 0
+        for idx in reversed(range(len(url_cols) + 1)):
+            from_idx = idx
+            if bool(self._request("HEAD", url_cols[idx - 1])):
+                break
+        for idx in range(from_idx, len(url_cols)):
+            response = self._request("MKCOL", url_cols[idx])
+            if response.status_code not in (200, 201):
+                if bool(self._request("HEAD", url_cols[idx])):
+                    continue
+                raise HTTPError(response.status_code, response.reason)
+
+    def gc(self):
+        raise NotImplementedError
+
+    def list_cache_paths(self, prefix=None, progress_callback=None):
+        raise NotImplementedError
+
+    def walk_files(self, path_info):
+        raise NotImplementedError

dvc/remote/webdavs.py

@@ -0,0 +1,15 @@
+from .webdav import RemoteWEBDAV
+from dvc.scheme import Schemes
+
+
+class RemoteWEBDAVS(RemoteWEBDAV):
+    scheme = Schemes.WEBDAVS
+
+    def gc(self):
+        raise NotImplementedError
+
+    def list_cache_paths(self, prefix=None, progress_callback=None):
+        raise NotImplementedError
+
+    def walk_files(self, path_info):
+        raise NotImplementedError

dvc/scheme.py

@@ -9,3 +9,5 @@ class Schemes:
     GDRIVE = "gdrive"
     LOCAL = "local"
     OSS = "oss"
+    WEBDAV = "webdav"
+    WEBDAVS = "webdavs"

tests/unit/remote/test_webdav.py

@@ -0,0 +1,20 @@
+import pytest
+
+from dvc.exceptions import HTTPError
+from dvc.path_info import WebdavURLInfo
+from dvc.remote.webdav import RemoteWEBDAV
+from tests.utils.httpd import StaticFileServer, WebDavSimpleHandler
+
+
+def test_create_collections(dvc):
+    with StaticFileServer(handler_class=WebDavSimpleHandler) as httpd:
+        url0 = "webdav://localhost:{}/a/b/file.txt".format(httpd.server_port)
+        url1 = "webdav://localhost:{}/a/c/file.txt".format(httpd.server_port)
+        config = {"url": url0}
+
+        remote = RemoteWEBDAV(dvc, config)
+
+        remote._create_collections(WebdavURLInfo(url0))
+
+        with pytest.raises(HTTPError):
+            remote._create_collections(WebdavURLInfo(url1))

tests/unit/test_path_info.py

@@ -7,6 +7,7 @@
 from dvc.path_info import HTTPURLInfo
 from dvc.path_info import PathInfo
 from dvc.path_info import URLInfo
+from dvc.path_info import WebdavURLInfo
 
 
 TEST_DEPTH = len(pathlib.Path(__file__).parents) + 1
@@ -89,3 +90,33 @@ def test_https_url_info_str():
 def test_path_info_as_posix(mocker, path, as_posix, osname):
     mocker.patch("os.name", osname)
     assert PathInfo(path).as_posix() == as_posix
+
+
+def test_webdav_url_info_str():
+    u1 = WebdavURLInfo("webdav://test.com/t1")
+    u2 = WebdavURLInfo("webdavs://test.com/t1")
+    assert u1.url == "http://test.com/t1"
+    assert u2.url == "https://test.com/t1"
+
+
+def test_webdav_collections_path():
+    u = WebdavURLInfo("webdav://test.com/t1")
+    assert u.get_collections() == []
+
+    u = WebdavURLInfo("webdav://test.com/")
+    assert u.get_collections() == []
+
+    u = WebdavURLInfo("webdav://test.com")
+    assert u.get_collections() == []
+
+    u = WebdavURLInfo("webdav://test.com/t1/")
+    assert u.get_collections() == ["http://test.com/t1/"]
+
+    u = WebdavURLInfo("webdav://test.com/t1/check")
+    assert u.get_collections() == ["http://test.com/t1/"]
+
+    u = WebdavURLInfo("webdav://test.com/t1/t2/check")
+    assert u.get_collections() == [
+        "http://test.com/t1/",
+        "http://test.com/t1/t2/",
+    ]

tests/utils/httpd.py

@@ -65,6 +65,28 @@ def do_POST(self):
         self.end_headers()
 
 
+class WebDavSimpleHandler(SimpleHTTPRequestHandler):
+    def do_HEAD(self):
+        if self.path == "/a/":
+            self.send_response(HTTPStatus.OK)
+        elif self.path == "/a/b/":
+            self.send_response(HTTPStatus.OK)
+        elif self.path == "/a/c/":
+            self.send_response(HTTPStatus.BAD_REQUEST)
+        else:
+            self.send_response(HTTPStatus.BAD_REQUEST)
+        self.end_headers()
+
+    def do_MKCOL(self):
+        if self.path == "/a/b/":
+            self.send_response(HTTPStatus.CREATED)
+        elif self.path == "/a/c/":
+            self.send_response(HTTPStatus.BAD_REQUEST)
+        else:
+            self.send_response(HTTPStatus.BAD_REQUEST)
+        self.end_headers()
+
+
 class StaticFileServer:
     _lock = threading.Lock()