Skip to content

Commit

Permalink
tests: cache test directories (iterative#4601)
Browse files Browse the repository at this point in the history
* tests: ignore more hdfs tests

* tests: remove unused mark

* gha: don't limit the number of test workers

* cache created test dirs
efiop authored Sep 23, 2020

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent e1b3444 commit 688da04
Showing 5 changed files with 55 additions and 18 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/tests.yaml
Original file line number Diff line number Diff line change
@@ -31,7 +31,7 @@ jobs:
git config --global user.email "dvctester@example.com"
git config --global user.name "DVC Tester"
- name: run tests
run: python -m tests -n=2 --cov-report=xml
run: python -m tests --cov-report=xml
- name: upload coverage report
if: github.event.name == 'push'
uses: codecov/codecov-action@v1.0.13
27 changes: 24 additions & 3 deletions tests/dir_helpers.py
Original file line number Diff line number Diff line change
@@ -253,12 +253,33 @@ class PosixTmpDir(TmpDir, pathlib.PurePosixPath):
pass


CACHE = {}


@pytest.fixture(scope="session")
def make_tmp_dir(tmp_path_factory, request):
def make(name, *, scm=False, dvc=False, **kwargs):
def make_tmp_dir(tmp_path_factory, request, worker_id):
def make(name, *, scm=False, dvc=False, subdir=False):
from dvc.repo import Repo
from dvc.scm.git import Git
from dvc.utils.fs import fs_copy

cache = CACHE.get((scm, dvc, subdir))
if not cache:
cache = tmp_path_factory.mktemp("dvc-test-cache" + worker_id)
TmpDir(cache).init(scm=scm, dvc=dvc, subdir=subdir)
CACHE[(scm, dvc, subdir)] = os.fspath(cache)
path = tmp_path_factory.mktemp(name) if isinstance(name, str) else name
for entry in os.listdir(cache):
# shutil.copytree's dirs_exist_ok is only available in >=3.8
fs_copy(os.path.join(cache, entry), os.path.join(path, entry))
new_dir = TmpDir(path)
new_dir.init(scm=scm, dvc=dvc, **kwargs)
str_path = os.fspath(new_dir)
if dvc:
new_dir.dvc = Repo(str_path)
if scm:
new_dir.scm = (
new_dir.dvc.scm if hasattr(new_dir, "dvc") else Git(str_path)
)
request.addfinalizer(new_dir.close)
return new_dir

36 changes: 24 additions & 12 deletions tests/func/test_api.py
Original file line number Diff line number Diff line change
@@ -84,10 +84,15 @@ def test_open(tmp_dir, dvc, remote):
"azure",
"gdrive",
"oss",
"hdfs",
"http",
]
],
pytest.param(
pytest.lazy_fixture("hdfs"),
marks=pytest.mark.xfail(
reason="https://github.com/iterative/dvc/issues/4418",
),
),
pytest.param(
pytest.lazy_fixture("ssh"),
marks=pytest.mark.xfail(
@@ -136,17 +141,24 @@ def test_open_granular(tmp_dir, dvc, remote):
@pytest.mark.parametrize(
"remote",
[
pytest.lazy_fixture(cloud)
for cloud in [
"real_s3", # NOTE: moto's s3 fails in some tests
"gs",
"azure",
"gdrive",
"oss",
"ssh",
"hdfs",
"http",
]
*[
pytest.lazy_fixture(cloud)
for cloud in [
"real_s3", # NOTE: moto's s3 fails in some tests
"gs",
"azure",
"gdrive",
"oss",
"ssh",
"http",
]
],
pytest.param(
pytest.lazy_fixture("hdfs"),
marks=pytest.mark.xfail(
reason="https://github.com/iterative/dvc/issues/4418",
),
),
],
indirect=True,
)
7 changes: 6 additions & 1 deletion tests/func/test_update.py
Original file line number Diff line number Diff line change
@@ -172,7 +172,12 @@ def test_update_before_and_after_dvc_init(tmp_dir, dvc, git_dir):
pytest.lazy_fixture("local_cloud"),
pytest.lazy_fixture("s3"),
pytest.lazy_fixture("gs"),
pytest.lazy_fixture("hdfs"),
pytest.param(
pytest.lazy_fixture("hdfs"),
marks=pytest.mark.xfail(
reason="https://github.com/iterative/dvc/issues/4418",
),
),
pytest.param(
pytest.lazy_fixture("ssh"),
marks=pytest.mark.skipif(
1 change: 0 additions & 1 deletion tests/unit/dependency/test_params.py
Original file line number Diff line number Diff line change
@@ -132,7 +132,6 @@ def test_get_hash_missing_param(tmp_dir, dvc):
dep.get_hash()


@pytest.mark.regression_4184
@pytest.mark.parametrize("param_value", ["", "false", "[]", "{}", "null"])
def test_params_with_false_values(tmp_dir, dvc, param_value):
"""These falsy params values should not ignored by `status` on loading."""

0 comments on commit 688da04

Please sign in to comment.