diff --git a/dev-requirements.txt b/dev-requirements.txt index a783d7f64a..9911ffbc22 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -14,7 +14,6 @@ attrs==20.3.0 # jsonschema # pytest # pytest-docker - # scantree backports.entry-points-selectable==1.1.0 # via virtualenv bcrypt==3.2.0 @@ -37,6 +36,10 @@ charset-normalizer==2.0.7 # via # -c requirements.txt # requests +checksumdir==1.2.0 + # via + # -c requirements.txt + # flytekit click==7.1.2 # via # -c requirements.txt @@ -66,10 +69,6 @@ deprecated==1.2.13 # via # -c requirements.txt # flytekit -dirhash==0.2.1 - # via - # -c requirements.txt - # flytekit diskcache==5.2.1 # via # -c requirements.txt @@ -187,10 +186,6 @@ paramiko==2.8.0 # via # -c requirements.txt # docker -pathspec==0.9.0 - # via - # -c requirements.txt - # scantree platformdirs==2.4.0 # via # -c requirements.txt @@ -264,7 +259,7 @@ pyyaml==5.4.1 # -c requirements.txt # docker-compose # pre-commit -regex==2021.10.8 +regex==2021.10.21 # via # -c requirements.txt # docker-image-py @@ -283,10 +278,6 @@ retry==0.9.2 # via # -c requirements.txt # flytekit -scantree==0.0.1 - # via - # -c requirements.txt - # dirhash secretstorage==3.3.1 # via # -c requirements.txt @@ -302,7 +293,6 @@ six==1.16.0 # pynacl # python-dateutil # responses - # scantree # virtualenv # websocket-client sortedcontainers==2.4.0 diff --git a/doc-requirements.txt b/doc-requirements.txt index 3bb6a7e372..02f4e05b0d 100644 --- a/doc-requirements.txt +++ b/doc-requirements.txt @@ -13,9 +13,7 @@ ansiwrap==0.8.4 astroid==2.8.3 # via sphinx-autoapi attrs==21.2.0 - # via - # jsonschema - # scantree + # via jsonschema babel==2.9.1 # via sphinx backcall==0.2.0 @@ -46,6 +44,8 @@ cffi==1.15.0 # pynacl charset-normalizer==2.0.7 # via requests +checksumdir==1.2.0 + # via flytekit click==7.1.2 # via # black @@ -71,8 +71,6 @@ defusedxml==0.7.1 # via nbconvert deprecated==1.2.13 # via flytekit -dirhash==0.2.1 - # via flytekit diskcache==5.2.1 # via flytekit docker-image-py==0.1.12 @@ -210,9 +208,7 @@ paramiko==2.8.0 parso==0.8.2 # via jedi pathspec==0.9.0 - # via - # black - # scantree + # via black pexpect==4.8.0 # via ipython pickleshare==0.7.5 @@ -278,7 +274,7 @@ pyyaml==6.0 # sphinx-autoapi pyzmq==22.3.0 # via jupyter-client -regex==2021.10.8 +regex==2021.10.21 # via # black # docker-image-py @@ -298,8 +294,6 @@ s3transfer==0.5.0 # via boto3 sagemaker-training==3.9.2 # via flytekit -scantree==0.0.1 - # via dirhash scipy==1.7.1 # via sagemaker-training secretstorage==3.3.1 @@ -315,7 +309,6 @@ six==1.16.0 # responses # retrying # sagemaker-training - # scantree # sphinx-code-include # thrift snowballstemmer==2.1.0 diff --git a/flytekit/tools/fast_registration.py b/flytekit/tools/fast_registration.py index 00ef36855e..f53a9b0b96 100644 --- a/flytekit/tools/fast_registration.py +++ b/flytekit/tools/fast_registration.py @@ -4,7 +4,7 @@ import tempfile as _tempfile from pathlib import Path as _Path -import dirhash as _dirhash +import checksumdir from flytekit.interfaces.data import data_proxy as _data_proxy from flytekit.interfaces.data.data_proxy import Data as _Data @@ -18,7 +18,7 @@ def compute_digest(source_dir: _os.PathLike) -> str: :param _os.PathLike source_dir: :return Text: """ - return f"fast{_dirhash.dirhash(source_dir, 'md5', match=['*.py'])}" + return f"fast{checksumdir.dirhash(source_dir, 'md5', include_paths=True)}" def _write_marker(marker: _os.PathLike): diff --git a/requirements-spark2.txt b/requirements-spark2.txt index 45e3ca4dfd..bcbfd29934 100644 --- a/requirements-spark2.txt +++ b/requirements-spark2.txt @@ -14,7 +14,6 @@ attrs==20.3.0 # via # -c requirements.in # jsonschema - # scantree backcall==0.2.0 # via ipython bcrypt==3.2.0 @@ -38,6 +37,8 @@ cffi==1.15.0 # pynacl charset-normalizer==2.0.7 # via requests +checksumdir==1.2.0 + # via flytekit click==7.1.2 # via # black @@ -60,8 +61,6 @@ defusedxml==0.7.1 # via nbconvert deprecated==1.2.13 # via flytekit -dirhash==0.2.1 - # via flytekit diskcache==5.2.1 # via flytekit docker-image-py==0.1.12 @@ -184,9 +183,7 @@ paramiko==2.8.0 parso==0.8.2 # via jedi pathspec==0.9.0 - # via - # black - # scantree + # via black pexpect==4.8.0 # via ipython pickleshare==0.7.5 @@ -247,7 +244,7 @@ pyyaml==5.4.1 # papermill pyzmq==22.3.0 # via jupyter-client -regex==2021.10.8 +regex==2021.10.21 # via # black # docker-image-py @@ -266,8 +263,6 @@ s3transfer==0.5.0 # via boto3 sagemaker-training==3.9.2 # via flytekit -scantree==0.0.1 - # via dirhash scipy==1.7.1 # via sagemaker-training secretstorage==3.3.1 @@ -284,7 +279,6 @@ six==1.16.0 # responses # retrying # sagemaker-training - # scantree # thrift sortedcontainers==2.4.0 # via flytekit diff --git a/requirements.txt b/requirements.txt index 30b5ae2e1c..929bd0c36e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,6 @@ attrs==20.3.0 # via # -r requirements.in # jsonschema - # scantree backcall==0.2.0 # via ipython bcrypt==3.2.0 @@ -36,6 +35,8 @@ cffi==1.15.0 # pynacl charset-normalizer==2.0.7 # via requests +checksumdir==1.2.0 + # via flytekit click==7.1.2 # via # black @@ -58,8 +59,6 @@ defusedxml==0.7.1 # via nbconvert deprecated==1.2.13 # via flytekit -dirhash==0.2.1 - # via flytekit diskcache==5.2.1 # via flytekit docker-image-py==0.1.12 @@ -182,9 +181,7 @@ paramiko==2.8.0 parso==0.8.2 # via jedi pathspec==0.9.0 - # via - # black - # scantree + # via black pexpect==4.8.0 # via ipython pickleshare==0.7.5 @@ -245,7 +242,7 @@ pyyaml==5.4.1 # papermill pyzmq==22.3.0 # via jupyter-client -regex==2021.10.8 +regex==2021.10.21 # via # black # docker-image-py @@ -264,8 +261,6 @@ s3transfer==0.5.0 # via boto3 sagemaker-training==3.9.2 # via flytekit -scantree==0.0.1 - # via dirhash scipy==1.7.1 # via sagemaker-training secretstorage==3.3.1 @@ -282,7 +277,6 @@ six==1.16.0 # responses # retrying # sagemaker-training - # scantree # thrift sortedcontainers==2.4.0 # via flytekit diff --git a/setup.py b/setup.py index 706b4dbbd8..6d33d34b13 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ packages=find_packages(exclude=["tests*"]), url="https://github.com/flyteorg/flytekit", description="Flyte SDK for Python", - long_description=open("README.md").read(), + long_description=open("README.md", encoding="utf-8").read(), long_description_content_type="text/markdown", entry_points={ "console_scripts": [ @@ -89,11 +89,11 @@ "dataclasses-json>=0.5.2", "marshmallow-jsonschema>=0.12.0", "natsort>=7.0.1", - "dirhash>=0.2.1", "docker-image-py>=0.1.10", "singledispatchmethod; python_version < '3.8.0'", "docstring-parser>=0.9.0", "diskcache>=5.2.1", + "checksumdir>=1.2.0", ], extras_require=extras_require, scripts=[ diff --git a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt index 9263f94fcc..bb746a6e9b 100644 --- a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt +++ b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt @@ -75,7 +75,7 @@ numpy==1.21.3 # opencv-python # pandas # pyarrow -opencv-python==4.5.3.56 +opencv-python==4.5.4.58 # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in pandas==1.3.4 # via flytekit @@ -109,7 +109,7 @@ pytz==2018.4 # via # flytekit # pandas -regex==2021.10.8 +regex==2021.10.21 # via docker-image-py requests==2.26.0 # via