azure: support directory imports with azure (#6245)

* azure: support directory imports with azure * upgrade azurite to 3.13 * tmp: use azurite 3.12? * upgrade to azurite 3.13.1 * bump adlfs to 2021.7.0
iterative · Jul 13, 2021 · 58f0911 · 58f0911
1 parent f45311e
commit 58f0911
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 17 deletions.
diff --git a/setup.py b/setup.py
@@ -99,7 +99,7 @@ def run(self):
 gs = ["gcsfs==2021.6.1"]
 gdrive = ["pydrive2>=1.8.1", "six >= 1.13.0"]
 s3 = ["s3fs==2021.6.1", "aiobotocore[boto3]==1.3.0"]
-azure = ["adlfs==0.7.1", "azure-identity>=1.4.0", "knack"]
+azure = ["adlfs==2021.7.0", "azure-identity>=1.4.0", "knack"]
 # https://github.com/Legrandin/pycryptodome/issues/465
 oss = ["oss2==2.6.1", "pycryptodome>=3.10"]
 ssh = ["paramiko[invoke]>=2.7.0"]

diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml
@@ -2,7 +2,7 @@
 version: '3.2'
 services:
   azurite:
-    image: mcr.microsoft.com/azure-storage/azurite:3.10.0
+    image: mcr.microsoft.com/azure-storage/azurite:3.13.1
     command: azurite -L -l /data --blobHost 0.0.0.0 --queueHost 0.0.0.0
     ports:
       - "10000"

diff --git a/tests/func/test_import_url.py b/tests/func/test_import_url.py
@@ -161,6 +161,12 @@ def test_import_url(tmp_dir, dvc, workspace):
             "2aa17f8daa26996b3f7a4cf8888ac9ac",
             "ec602a6ba97b2dd07bd6d2cd89674a60.dir",
         ),
+        (
+            pytest.lazy_fixture("azure"),
+            None,  # ETags for azure are not consistent with the actual
+            None,  # content of the object, so they will change every time
+            # we create new objects. We'll skip stage content check for Azure.
+        ),
         (
             pytest.lazy_fixture("hdfs"),
             "ec0943f83357f702033c98e70b853c8c",
@@ -192,23 +198,24 @@ def test_import_url_dir(tmp_dir, dvc, workspace, stage_md5, dir_md5):
     assert list(os.listdir(tmp_dir / "dir" / "subdir")) == ["subfile"]
     assert (tmp_dir / "dir" / "subdir" / "subfile").read_text() == "subfile"
 
-    assert (tmp_dir / "dir.dvc").read_text() == (
-        f"md5: {stage_md5}\n"
-        "frozen: true\n"
-        "deps:\n"
-        f"- md5: {dir_md5}\n"
-        "  size: 11\n"
-        "  nfiles: 2\n"
-        "  path: remote://workspace/dir\n"
-        "outs:\n"
-        "- md5: b6dcab6ccd17ca0a8bf4a215a37d14cc.dir\n"
-        "  size: 11\n"
-        "  nfiles: 2\n"
-        "  path: dir\n"
-    )
-
     assert dvc.status() == {}
 
+    if stage_md5 is not None and dir_md5 is not None:
+        assert (tmp_dir / "dir.dvc").read_text() == (
+            f"md5: {stage_md5}\n"
+            "frozen: true\n"
+            "deps:\n"
+            f"- md5: {dir_md5}\n"
+            "  size: 11\n"
+            "  nfiles: 2\n"
+            "  path: remote://workspace/dir\n"
+            "outs:\n"
+            "- md5: b6dcab6ccd17ca0a8bf4a215a37d14cc.dir\n"
+            "  size: 11\n"
+            "  nfiles: 2\n"
+            "  path: dir\n"
+        )
+
 
 def test_import_url_preserve_meta(tmp_dir, dvc):
     text = textwrap.dedent(