Change dirhash package to checksumdir (#708)

Signed-off-by: Eduardo Apolinario <[email protected]> Signed-off-by: Martin Lyrå <[email protected]>
flyteorg · Oct 22, 2021 · b48359d · b48359d
1 parent 36cc418
commit b48359d
Show file tree

Hide file tree

Showing 7 changed files with 24 additions and 53 deletions.
diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -14,7 +14,6 @@ attrs==20.3.0
     #   jsonschema
     #   pytest
     #   pytest-docker
-    #   scantree
 backports.entry-points-selectable==1.1.0
     # via virtualenv
 bcrypt==3.2.0
@@ -37,6 +36,10 @@ charset-normalizer==2.0.7
     # via
     #   -c requirements.txt
     #   requests
+checksumdir==1.2.0
+    # via
+    #   -c requirements.txt
+    #   flytekit
 click==7.1.2
     # via
     #   -c requirements.txt
@@ -66,10 +69,6 @@ deprecated==1.2.13
     # via
     #   -c requirements.txt
     #   flytekit
-dirhash==0.2.1
-    # via
-    #   -c requirements.txt
-    #   flytekit
 diskcache==5.2.1
     # via
     #   -c requirements.txt
@@ -187,10 +186,6 @@ paramiko==2.8.0
     # via
     #   -c requirements.txt
     #   docker
-pathspec==0.9.0
-    # via
-    #   -c requirements.txt
-    #   scantree
 platformdirs==2.4.0
     # via
     #   -c requirements.txt
@@ -264,7 +259,7 @@ pyyaml==5.4.1
     #   -c requirements.txt
     #   docker-compose
     #   pre-commit
-regex==2021.10.8
+regex==2021.10.21
     # via
     #   -c requirements.txt
     #   docker-image-py
@@ -283,10 +278,6 @@ retry==0.9.2
     # via
     #   -c requirements.txt
     #   flytekit
-scantree==0.0.1
-    # via
-    #   -c requirements.txt
-    #   dirhash
 secretstorage==3.3.1
     # via
     #   -c requirements.txt
@@ -302,7 +293,6 @@ six==1.16.0
     #   pynacl
     #   python-dateutil
     #   responses
-    #   scantree
     #   virtualenv
     #   websocket-client
 sortedcontainers==2.4.0

diff --git a/doc-requirements.txt b/doc-requirements.txt
@@ -13,9 +13,7 @@ ansiwrap==0.8.4
 astroid==2.8.3
     # via sphinx-autoapi
 attrs==21.2.0
-    # via
-    #   jsonschema
-    #   scantree
+    # via jsonschema
 babel==2.9.1
     # via sphinx
 backcall==0.2.0
@@ -46,6 +44,8 @@ cffi==1.15.0
     #   pynacl
 charset-normalizer==2.0.7
     # via requests
+checksumdir==1.2.0
+    # via flytekit
 click==7.1.2
     # via
     #   black
@@ -71,8 +71,6 @@ defusedxml==0.7.1
     # via nbconvert
 deprecated==1.2.13
     # via flytekit
-dirhash==0.2.1
-    # via flytekit
 diskcache==5.2.1
     # via flytekit
 docker-image-py==0.1.12
@@ -210,9 +208,7 @@ paramiko==2.8.0
 parso==0.8.2
     # via jedi
 pathspec==0.9.0
-    # via
-    #   black
-    #   scantree
+    # via black
 pexpect==4.8.0
     # via ipython
 pickleshare==0.7.5
@@ -278,7 +274,7 @@ pyyaml==6.0
     #   sphinx-autoapi
 pyzmq==22.3.0
     # via jupyter-client
-regex==2021.10.8
+regex==2021.10.21
     # via
     #   black
     #   docker-image-py
@@ -298,8 +294,6 @@ s3transfer==0.5.0
     # via boto3
 sagemaker-training==3.9.2
     # via flytekit
-scantree==0.0.1
-    # via dirhash
 scipy==1.7.1
     # via sagemaker-training
 secretstorage==3.3.1
@@ -315,7 +309,6 @@ six==1.16.0
     #   responses
     #   retrying
     #   sagemaker-training
-    #   scantree
     #   sphinx-code-include
     #   thrift
 snowballstemmer==2.1.0

diff --git a/flytekit/tools/fast_registration.py b/flytekit/tools/fast_registration.py
@@ -4,7 +4,7 @@
 import tempfile as _tempfile
 from pathlib import Path as _Path
 
-import dirhash as _dirhash
+import checksumdir
 
 from flytekit.interfaces.data import data_proxy as _data_proxy
 from flytekit.interfaces.data.data_proxy import Data as _Data
@@ -18,7 +18,7 @@ def compute_digest(source_dir: _os.PathLike) -> str:
     :param _os.PathLike source_dir:
     :return Text:
     """
-    return f"fast{_dirhash.dirhash(source_dir, 'md5', match=['*.py'])}"
+    return f"fast{checksumdir.dirhash(source_dir, 'md5', include_paths=True)}"
 
 
 def _write_marker(marker: _os.PathLike):

diff --git a/requirements-spark2.txt b/requirements-spark2.txt
@@ -14,7 +14,6 @@ attrs==20.3.0
     # via
     #   -c requirements.in
     #   jsonschema
-    #   scantree
 backcall==0.2.0
     # via ipython
 bcrypt==3.2.0
@@ -38,6 +37,8 @@ cffi==1.15.0
     #   pynacl
 charset-normalizer==2.0.7
     # via requests
+checksumdir==1.2.0
+    # via flytekit
 click==7.1.2
     # via
     #   black
@@ -60,8 +61,6 @@ defusedxml==0.7.1
     # via nbconvert
 deprecated==1.2.13
     # via flytekit
-dirhash==0.2.1
-    # via flytekit
 diskcache==5.2.1
     # via flytekit
 docker-image-py==0.1.12
@@ -184,9 +183,7 @@ paramiko==2.8.0
 parso==0.8.2
     # via jedi
 pathspec==0.9.0
-    # via
-    #   black
-    #   scantree
+    # via black
 pexpect==4.8.0
     # via ipython
 pickleshare==0.7.5
@@ -247,7 +244,7 @@ pyyaml==5.4.1
     #   papermill
 pyzmq==22.3.0
     # via jupyter-client
-regex==2021.10.8
+regex==2021.10.21
     # via
     #   black
     #   docker-image-py
@@ -266,8 +263,6 @@ s3transfer==0.5.0
     # via boto3
 sagemaker-training==3.9.2
     # via flytekit
-scantree==0.0.1
-    # via dirhash
 scipy==1.7.1
     # via sagemaker-training
 secretstorage==3.3.1
@@ -284,7 +279,6 @@ six==1.16.0
     #   responses
     #   retrying
     #   sagemaker-training
-    #   scantree
     #   thrift
 sortedcontainers==2.4.0
     # via flytekit

diff --git a/requirements.txt b/requirements.txt
@@ -12,7 +12,6 @@ attrs==20.3.0
     # via
     #   -r requirements.in
     #   jsonschema
-    #   scantree
 backcall==0.2.0
     # via ipython
 bcrypt==3.2.0
@@ -36,6 +35,8 @@ cffi==1.15.0
     #   pynacl
 charset-normalizer==2.0.7
     # via requests
+checksumdir==1.2.0
+    # via flytekit
 click==7.1.2
     # via
     #   black
@@ -58,8 +59,6 @@ defusedxml==0.7.1
     # via nbconvert
 deprecated==1.2.13
     # via flytekit
-dirhash==0.2.1
-    # via flytekit
 diskcache==5.2.1
     # via flytekit
 docker-image-py==0.1.12
@@ -182,9 +181,7 @@ paramiko==2.8.0
 parso==0.8.2
     # via jedi
 pathspec==0.9.0
-    # via
-    #   black
-    #   scantree
+    # via black
 pexpect==4.8.0
     # via ipython
 pickleshare==0.7.5
@@ -245,7 +242,7 @@ pyyaml==5.4.1
     #   papermill
 pyzmq==22.3.0
     # via jupyter-client
-regex==2021.10.8
+regex==2021.10.21
     # via
     #   black
     #   docker-image-py
@@ -264,8 +261,6 @@ s3transfer==0.5.0
     # via boto3
 sagemaker-training==3.9.2
     # via flytekit
-scantree==0.0.1
-    # via dirhash
 scipy==1.7.1
     # via sagemaker-training
 secretstorage==3.3.1
@@ -282,7 +277,6 @@ six==1.16.0
     #   responses
     #   retrying
     #   sagemaker-training
-    #   scantree
     #   thrift
 sortedcontainers==2.4.0
     # via flytekit

diff --git a/setup.py b/setup.py
@@ -52,7 +52,7 @@
     packages=find_packages(exclude=["tests*"]),
     url="https://github.com/flyteorg/flytekit",
     description="Flyte SDK for Python",
-    long_description=open("README.md").read(),
+    long_description=open("README.md", encoding="utf-8").read(),
     long_description_content_type="text/markdown",
     entry_points={
         "console_scripts": [
@@ -89,11 +89,11 @@
         "dataclasses-json>=0.5.2",
         "marshmallow-jsonschema>=0.12.0",
         "natsort>=7.0.1",
-        "dirhash>=0.2.1",
         "docker-image-py>=0.1.10",
         "singledispatchmethod; python_version < '3.8.0'",
         "docstring-parser>=0.9.0",
         "diskcache>=5.2.1",
+        "checksumdir>=1.2.0",
     ],
     extras_require=extras_require,
     scripts=[

diff --git a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt
@@ -75,7 +75,7 @@ numpy==1.21.3
     #   opencv-python
     #   pandas
     #   pyarrow
-opencv-python==4.5.3.56
+opencv-python==4.5.4.58
     # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in
 pandas==1.3.4
     # via flytekit
@@ -109,7 +109,7 @@ pytz==2018.4
     # via
     #   flytekit
     #   pandas
-regex==2021.10.8
+regex==2021.10.21
     # via docker-image-py
 requests==2.26.0
     # via