Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

remote: add support for WebDAV #3647

Closed
wants to merge 16 commits into from
Closed
4 changes: 4 additions & 0 deletions dvc/remote/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@
from dvc.remote.oss import RemoteOSS
from dvc.remote.s3 import RemoteS3
from dvc.remote.ssh import RemoteSSH
from dvc.remote.webdav import RemoteWEBDAV
from dvc.remote.webdavs import RemoteWEBDAVS


REMOTES = [
Expand All @@ -23,6 +25,8 @@
RemoteS3,
RemoteSSH,
RemoteOSS,
RemoteWEBDAV,
RemoteWEBDAVS,
# NOTE: RemoteLOCAL is the default
]

Expand Down
8 changes: 4 additions & 4 deletions dvc/remote/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -552,8 +552,8 @@ def upload(self, from_info, to_info, name=None, no_progress_bar=False):
if not hasattr(self, "_upload"):
raise RemoteActionNotImplemented("upload", self.scheme)

if to_info.scheme != self.scheme:
raise NotImplementedError
# if to_info.scheme != self.scheme:
# raise NotImplementedError
shizacat marked this conversation as resolved.
Show resolved Hide resolved

if from_info.scheme != "local":
raise NotImplementedError
Expand Down Expand Up @@ -588,8 +588,8 @@ def download(
if not hasattr(self, "_download"):
raise RemoteActionNotImplemented("download", self.scheme)

if from_info.scheme != self.scheme:
raise NotImplementedError
# if from_info.scheme != self.scheme:
# raise NotImplementedError

if to_info.scheme == self.scheme != "local":
self.copy(from_info, to_info)
Expand Down
54 changes: 54 additions & 0 deletions dvc/remote/webdav.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
from .http import RemoteHTTP
from dvc.scheme import Schemes

import os.path

from dvc.progress import Tqdm
from dvc.exceptions import HTTPError


class RemoteWEBDAV(RemoteHTTP):
scheme = Schemes.WEBDAV

def __init__(self, repo, config):
super().__init__(repo, config)

url = config.get("url")
if url:
self.path_info = self.path_cls(url)
self.path_info.scheme = self.path_info.scheme.replace(
shizacat marked this conversation as resolved.
Show resolved Hide resolved
"webdav", "http")
user = config.get("user", None)
if user:
self.path_info.user = user
else:
self.path_info = None

self.auth = config.get("auth", None)
self.custom_auth_header = config.get("custom_auth_header", None)
self.password = config.get("password", None)
self.ask_password = config.get("ask_password", False)
self.headers = {}

def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
def chunks():
with open(from_file, "rb") as fd:
with Tqdm.wrapattr(
fd,
"read",
total=None
if no_progress_bar
else os.path.getsize(from_file),
leave=False,
desc=to_info.url if name is None else name,
disable=no_progress_bar,
) as fd_wrapped:
while True:
chunk = fd_wrapped.read(self.CHUNK_SIZE)
if not chunk:
break
yield chunk

response = self._request("PUT", to_info.url, data=chunks())
shizacat marked this conversation as resolved.
Show resolved Hide resolved
if response.status_code not in (200, 201):
raise HTTPError(response.status_code, response.reason)
6 changes: 6 additions & 0 deletions dvc/remote/webdavs.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
from .webdav import RemoteWEBDAV
from dvc.scheme import Schemes


class RemoteWEBDAVS(RemoteWEBDAV):
scheme = Schemes.WEBDAVS
2 changes: 2 additions & 0 deletions dvc/scheme.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,5 @@ class Schemes:
GDRIVE = "gdrive"
LOCAL = "local"
OSS = "oss"
WEBDAV = "webdav"
WEBDAVS = "webdavs"