Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

import: fix importing into subdirectory bug #4340

Merged
merged 5 commits into from
Aug 6, 2020
Merged

import: fix importing into subdirectory bug #4340

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
dfc2cf7
tests: test importing into subdirectory
pmrowla Aug 5, 2020
d9b7e8f
import: fix bug when output subdir doesnt already exist
pmrowla Aug 5, 2020
d4de505
use urlparse to determine if output path is local, not os.path.exists
pmrowla Aug 5, 2020
7c51b2f
account for windows drive letters
pmrowla Aug 5, 2020
e432438
import: error out if output subdirectory does not exist
pmrowla Aug 6, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 8 additions & 5 deletions dvc/utils/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -363,19 +363,22 @@ def resolve_output(inp, out):


def resolve_paths(repo, out):
from urllib.parse import urlparse
from ..dvcfile import DVC_FILE_SUFFIX
from ..path_info import PathInfo
from .fs import contains_symlink_up_to

abspath = os.path.abspath(out)
abspath = PathInfo(os.path.abspath(out))
dirname = os.path.dirname(abspath)
base = os.path.basename(os.path.normpath(out))

# NOTE: `out` might not exist yet, so using `dirname`(aka `wdir`) to check
# if it is a local path.
scheme = urlparse(out).scheme
if os.name == "nt" and scheme == abspath.drive[0].lower():
# urlparse interprets windows drive letters as URL scheme
scheme = ""
if (
os.path.exists(dirname) # out might not exist yet, so
and PathInfo(abspath).isin_or_eq(repo.root_dir)
not scheme
and abspath.isin_or_eq(repo.root_dir)
Comment on lines +375 to +381
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

out may be a local path or a remote://... url, but os.path.exists isn't a good enough check for whether or not out is a valid local path.

In the case where we are importing with -o subdir/file, if subdir doesn't already exist it is still a valid local output path (and we need to makedirs(subdir) later)

and not contains_symlink_up_to(abspath, repo.root_dir)
):
wdir = dirname
Expand Down
23 changes: 23 additions & 0 deletions tests/func/test_import.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from dvc.config import NoRemoteError
from dvc.dvcfile import Dvcfile
from dvc.exceptions import DownloadError, PathMissingError
from dvc.stage.exceptions import StagePathNotFoundError
from dvc.system import System
from dvc.utils.fs import makedirs, remove

Expand Down Expand Up @@ -132,6 +133,28 @@ def test_import_file_from_dir(tmp_dir, scm, dvc, erepo_dir):
assert (tmp_dir / "X.dvc").exists()


def test_import_file_from_dir_to_dir(tmp_dir, scm, dvc, erepo_dir):
with erepo_dir.chdir():
erepo_dir.dvc_gen({"dir": {"foo": "foo"}}, commit="create dir")

with pytest.raises(StagePathNotFoundError):
dvc.imp(
os.fspath(erepo_dir),
os.path.join("dir", "foo"),
out=os.path.join("dir", "foo"),
)

tmp_dir.gen({"dir": {}})
dvc.imp(
os.fspath(erepo_dir),
os.path.join("dir", "foo"),
out=os.path.join("dir", "foo"),
)
assert not (tmp_dir / "foo.dvc").exists()
assert (tmp_dir / "dir" / "foo").read_text() == "foo"
assert (tmp_dir / "dir" / "foo.dvc").exists()


def test_import_non_cached(erepo_dir, tmp_dir, dvc, scm):
src = "non_cached_output"
dst = src + "_imported"
Expand Down