diff --git a/Makefile b/Makefile index f2e8f5e80d..959da5c553 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ export REPOSITORY=flytekit PIP_COMPILE = pip-compile --upgrade --verbose +MOCK_FLYTE_REPO=tests/flytekit/integration/remote/mock_flyte_repo/workflows .SILENT: help .PHONY: help @@ -70,8 +71,12 @@ doc-requirements.txt: export CUSTOM_COMPILE_COMMAND := make doc-requirements.txt doc-requirements.txt: doc-requirements.in install-piptools $(PIP_COMPILE) $< +${MOCK_FLYTE_REPO}/requirements.txt: export CUSTOM_COMPILE_COMMAND := make ${MOCK_FLYTE_REPO}/requirements.txt +${MOCK_FLYTE_REPO}/requirements.txt: ${MOCK_FLYTE_REPO}/requirements.in install-piptools + $(PIP_COMPILE) $< + .PHONY: requirements -requirements: requirements.txt dev-requirements.txt requirements-spark2.txt doc-requirements.txt ## Compile requirements +requirements: requirements.txt dev-requirements.txt requirements-spark2.txt doc-requirements.txt ${MOCK_FLYTE_REPO}/requirements.txt ## Compile requirements # TODO: Change this in the future to be all of flytekit .PHONY: coverage diff --git a/dev-requirements.in b/dev-requirements.in index bf2f96d992..9743d0fc20 100644 --- a/dev-requirements.in +++ b/dev-requirements.in @@ -2,6 +2,7 @@ git+git://github.com/flyteorg/pytest-flyte@main#egg=pytest-flyte coverage[toml] +joblib mock pytest mypy diff --git a/dev-requirements.txt b/dev-requirements.txt index 67a2eac899..4e84560733 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -33,7 +33,7 @@ cffi==1.14.6 # pynacl cfgv==3.3.1 # via pre-commit -charset-normalizer==2.0.4 +charset-normalizer==2.0.6 # via # -c requirements.txt # requests @@ -53,11 +53,11 @@ cryptography==3.4.8 # via # -c requirements.txt # paramiko -dataclasses-json==0.5.5 +dataclasses-json==0.5.6 # via # -c requirements.txt # flytekit -decorator==5.0.9 +decorator==5.1.0 # via # -c requirements.txt # retry @@ -73,7 +73,7 @@ diskcache==5.2.1 # via # -c requirements.txt # flytekit -distlib==0.3.2 +distlib==0.3.3 # via virtualenv distro==1.6.0 # via docker-compose @@ -95,17 +95,17 @@ docstring-parser==0.10 # via # -c requirements.txt # flytekit -filelock==3.0.12 +filelock==3.1.0 # via virtualenv -flyteidl==0.21.0 +flyteidl==0.21.2 # via # -c requirements.txt # flytekit -grpcio==1.40.0 +grpcio==1.41.0 # via # -c requirements.txt # flytekit -identify==2.2.14 +identify==2.2.15 # via pre-commit idna==3.2 # via @@ -121,11 +121,13 @@ jinja2==3.0.1 # via # -c requirements.txt # pytest-flyte +joblib==1.0.1 + # via -r dev-requirements.in jsonschema==3.2.0 # via # -c requirements.txt # docker-compose -keyring==23.1.0 +keyring==23.2.1 # via # -c requirements.txt # flytekit @@ -171,7 +173,7 @@ packaging==21.0 # via # -c requirements.txt # pytest -pandas==1.3.2 +pandas==1.3.3 # via # -c requirements.txt # flytekit @@ -183,7 +185,7 @@ pathspec==0.9.0 # via # -c requirements.txt # scantree -platformdirs==2.3.0 +platformdirs==2.4.0 # via # -c requirements.txt # virtualenv @@ -191,7 +193,7 @@ pluggy==1.0.0 # via pytest pre-commit==2.15.0 # via -r dev-requirements.in -protobuf==3.17.3 +protobuf==3.18.0 # via # -c requirements.txt # flyteidl @@ -228,7 +230,7 @@ pytest==6.2.5 # pytest-flyte pytest-docker==0.10.3 # via pytest-flyte -git+git://github.com/flyteorg/pytest-flyte@main#egg=pytest-flyte +pytest-flyte @ git+git://github.com/flyteorg/pytest-flyte@main # via -r dev-requirements.in python-dateutil==2.8.1 # via @@ -256,7 +258,7 @@ pyyaml==5.4.1 # -c requirements.txt # docker-compose # pre-commit -regex==2021.8.28 +regex==2021.9.24 # via # -c requirements.txt # docker-image-py @@ -267,7 +269,7 @@ requests==2.26.0 # docker-compose # flytekit # responses -responses==0.13.4 +responses==0.14.0 # via # -c requirements.txt # flytekit @@ -287,7 +289,6 @@ six==1.16.0 # flytekit # grpcio # jsonschema - # protobuf # pynacl # python-dateutil # responses @@ -319,13 +320,13 @@ typing-inspect==0.7.1 # via # -c requirements.txt # dataclasses-json -urllib3==1.26.6 +urllib3==1.26.7 # via # -c requirements.txt # flytekit # requests # responses -virtualenv==20.7.2 +virtualenv==20.8.1 # via pre-commit websocket-client==0.59.0 # via @@ -340,7 +341,7 @@ wrapt==1.12.1 # -c requirements.txt # deprecated # flytekit -zipp==3.5.0 +zipp==3.6.0 # via # -c requirements.txt # importlib-metadata diff --git a/doc-requirements.txt b/doc-requirements.txt index e1d9448ede..ba238ed7d7 100644 --- a/doc-requirements.txt +++ b/doc-requirements.txt @@ -14,7 +14,7 @@ appnope==0.1.2 # via # ipykernel # ipython -astroid==2.7.3 +astroid==2.8.0 # via sphinx-autoapi attrs==21.2.0 # via @@ -31,13 +31,13 @@ beautifulsoup4==4.10.0 # furo # sphinx-code-include # sphinx-material -black==21.8b0 +black==21.9b0 # via papermill bleach==4.1.0 # via nbconvert -boto3==1.18.39 +boto3==1.18.50 # via sagemaker-training -botocore==1.21.39 +botocore==1.21.50 # via # boto3 # s3transfer @@ -48,7 +48,7 @@ cffi==1.14.6 # bcrypt # cryptography # pynacl -charset-normalizer==2.0.4 +charset-normalizer==2.0.6 # via requests click==7.1.2 # via @@ -64,9 +64,9 @@ cryptography==3.4.8 # paramiko css-html-js-minify==2.5.5 # via sphinx-material -dataclasses-json==0.5.5 +dataclasses-json==0.5.6 # via flytekit -decorator==5.0.9 +decorator==5.1.0 # via # ipython # retry @@ -89,15 +89,15 @@ entrypoints==0.3 # jupyter-client # nbconvert # papermill -flyteidl==0.21.0 +flyteidl==0.21.2 # via flytekit -git+git://github.com/flyteorg/furo@main +furo @ git+git://github.com/flyteorg/furo@main # via -r doc-requirements.in gevent==21.8.0 # via sagemaker-training -greenlet==1.1.1 +greenlet==1.1.2 # via gevent -grpcio==1.40.0 +grpcio==1.41.0 # via # -r doc-requirements.in # flytekit @@ -113,7 +113,7 @@ inotify_simple==1.2.1 # via sagemaker-training ipykernel==5.5.5 # via flytekit -ipython==7.27.0 +ipython==7.28.0 # via ipykernel ipython-genutils==0.2.0 # via nbformat @@ -130,11 +130,11 @@ jmespath==0.10.0 # botocore jsonschema==3.2.0 # via nbformat -jupyter-client==7.0.2 +jupyter-client==7.0.5 # via # ipykernel # nbclient -jupyter-core==4.7.1 +jupyter-core==4.8.1 # via # jupyter-client # nbconvert @@ -143,7 +143,7 @@ jupyterlab-pygments==0.1.2 # via nbconvert k8s-proto==0.0.3 # via flytekit -keyring==23.1.0 +keyring==23.2.1 # via flytekit lazy-object-proxy==1.6.0 # via astroid @@ -174,7 +174,7 @@ nbclient==0.5.4 # via # nbconvert # papermill -nbconvert==6.1.0 +nbconvert==6.2.0 # via flytekit nbformat==5.1.3 # via @@ -196,9 +196,9 @@ packaging==21.0 # via # bleach # sphinx -pandas==1.3.2 +pandas==1.3.3 # via flytekit -pandocfilters==1.4.3 +pandocfilters==1.5.0 # via nbconvert papermill==2.3.3 # via flytekit @@ -214,11 +214,11 @@ pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -platformdirs==2.3.0 +platformdirs==2.4.0 # via black prompt-toolkit==3.0.20 # via ipython -protobuf==3.17.3 +protobuf==3.18.0 # via # flyteidl # flytekit @@ -273,9 +273,9 @@ pyyaml==5.4.1 # via # papermill # sphinx-autoapi -pyzmq==22.2.1 +pyzmq==22.3.0 # via jupyter-client -regex==2021.8.28 +regex==2021.9.24 # via # black # docker-image-py @@ -285,7 +285,7 @@ requests==2.26.0 # papermill # responses # sphinx -responses==0.13.4 +responses==0.14.0 # via flytekit retry==0.9.2 # via flytekit @@ -306,7 +306,6 @@ six==1.16.0 # flytekit # grpcio # jsonschema - # protobuf # pynacl # python-dateutil # responses @@ -321,7 +320,7 @@ sortedcontainers==2.4.0 # via flytekit soupsieve==2.2.1 # via beautifulsoup4 -sphinx==4.1.2 +sphinx==4.2.0 # via # -r doc-requirements.in # furo @@ -341,7 +340,7 @@ sphinx-copybutton==0.4.0 # via -r doc-requirements.in sphinx-fontawesome==0.0.6 # via -r doc-requirements.in -sphinx-gallery==0.9.0 +sphinx-gallery==0.10.0 # via -r doc-requirements.in sphinx-material==0.0.34 # via -r doc-requirements.in @@ -371,7 +370,7 @@ text-unidecode==1.3 # via python-slugify textwrap3==0.9.2 # via ansiwrap -thrift==0.13.0 +thrift==0.15.0 # via hmsclient tomli==1.2.1 # via black @@ -379,7 +378,7 @@ tornado==6.1 # via # ipykernel # jupyter-client -tqdm==4.62.2 +tqdm==4.62.3 # via papermill traitlets==5.1.0 # via @@ -393,15 +392,16 @@ traitlets==5.1.0 # nbformat typing-extensions==3.10.0.2 # via + # astroid # black # typing-inspect typing-inspect==0.7.1 # via dataclasses-json -unidecode==1.3.1 +unidecode==1.3.2 # via # python-slugify # sphinx-autoapi -urllib3==1.26.6 +urllib3==1.26.7 # via # botocore # flytekit @@ -420,7 +420,7 @@ wrapt==1.12.1 # astroid # deprecated # flytekit -zipp==3.5.0 +zipp==3.6.0 # via importlib-metadata zope.event==4.5.0 # via gevent diff --git a/requirements-spark2.txt b/requirements-spark2.txt index 4007626c78..14353e8737 100644 --- a/requirements-spark2.txt +++ b/requirements-spark2.txt @@ -21,13 +21,13 @@ backcall==0.2.0 # via ipython bcrypt==3.2.0 # via paramiko -black==21.8b0 +black==21.9b0 # via papermill bleach==4.1.0 # via nbconvert -boto3==1.18.39 +boto3==1.18.50 # via sagemaker-training -botocore==1.21.39 +botocore==1.21.50 # via # boto3 # s3transfer @@ -38,7 +38,7 @@ cffi==1.14.6 # bcrypt # cryptography # pynacl -charset-normalizer==2.0.4 +charset-normalizer==2.0.6 # via requests click==7.1.2 # via @@ -50,9 +50,9 @@ croniter==1.0.15 # via flytekit cryptography==3.4.8 # via paramiko -dataclasses-json==0.5.5 +dataclasses-json==0.5.6 # via flytekit -decorator==5.0.9 +decorator==5.1.0 # via # ipython # retry @@ -73,13 +73,13 @@ entrypoints==0.3 # jupyter-client # nbconvert # papermill -flyteidl==0.21.0 +flyteidl==0.21.2 # via flytekit gevent==21.8.0 # via sagemaker-training -greenlet==1.1.1 +greenlet==1.1.2 # via gevent -grpcio==1.40.0 +grpcio==1.41.0 # via flytekit hmsclient==0.1.1 # via flytekit @@ -91,7 +91,7 @@ inotify_simple==1.2.1 # via sagemaker-training ipykernel==5.5.5 # via flytekit -ipython==7.27.0 +ipython==7.28.0 # via ipykernel ipython-genutils==0.2.0 # via nbformat @@ -105,11 +105,11 @@ jmespath==0.10.0 # botocore jsonschema==3.2.0 # via nbformat -jupyter-client==7.0.2 +jupyter-client==7.0.5 # via # ipykernel # nbclient -jupyter-core==4.7.1 +jupyter-core==4.8.1 # via # jupyter-client # nbconvert @@ -118,7 +118,7 @@ jupyterlab-pygments==0.1.2 # via nbconvert k8s-proto==0.0.3 # via flytekit -keyring==23.1.0 +keyring==23.2.1 # via flytekit markupsafe==2.0.1 # via jinja2 @@ -145,7 +145,7 @@ nbclient==0.5.4 # via # nbconvert # papermill -nbconvert==6.1.0 +nbconvert==6.2.0 # via flytekit nbformat==5.1.3 # via @@ -165,9 +165,9 @@ numpy==1.21.2 # scipy packaging==21.0 # via bleach -pandas==1.3.2 +pandas==1.3.3 # via flytekit -pandocfilters==1.4.3 +pandocfilters==1.5.0 # via nbconvert papermill==2.3.3 # via flytekit @@ -183,11 +183,11 @@ pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -platformdirs==2.3.0 +platformdirs==2.4.0 # via black prompt-toolkit==3.0.20 # via ipython -protobuf==3.17.3 +protobuf==3.18.0 # via # flyteidl # flytekit @@ -235,9 +235,9 @@ pytz==2018.4 # pandas pyyaml==5.4.1 # via papermill -pyzmq==22.2.1 +pyzmq==22.3.0 # via jupyter-client -regex==2021.8.28 +regex==2021.9.24 # via # black # docker-image-py @@ -246,7 +246,7 @@ requests==2.26.0 # flytekit # papermill # responses -responses==0.13.4 +responses==0.14.0 # via flytekit retry==0.9.2 # via flytekit @@ -267,7 +267,6 @@ six==1.16.0 # flytekit # grpcio # jsonschema - # protobuf # pynacl # python-dateutil # responses @@ -285,7 +284,7 @@ testpath==0.5.0 # via nbconvert textwrap3==0.9.2 # via ansiwrap -thrift==0.13.0 +thrift==0.15.0 # via hmsclient tomli==1.2.1 # via black @@ -293,7 +292,7 @@ tornado==6.1 # via # ipykernel # jupyter-client -tqdm==4.62.2 +tqdm==4.62.3 # via papermill traitlets==5.1.0 # via @@ -311,7 +310,7 @@ typing-extensions==3.10.0.2 # typing-inspect typing-inspect==0.7.1 # via dataclasses-json -urllib3==1.26.6 +urllib3==1.26.7 # via # botocore # flytekit @@ -329,7 +328,7 @@ wrapt==1.12.1 # via # deprecated # flytekit -zipp==3.5.0 +zipp==3.6.0 # via importlib-metadata zope.event==4.5.0 # via gevent diff --git a/requirements.txt b/requirements.txt index d6d855e162..bcff3e9109 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,13 +21,13 @@ backcall==0.2.0 # via ipython bcrypt==3.2.0 # via paramiko -black==21.8b0 +black==21.9b0 # via papermill bleach==4.1.0 # via nbconvert -boto3==1.18.39 +boto3==1.18.50 # via sagemaker-training -botocore==1.21.39 +botocore==1.21.50 # via # boto3 # s3transfer @@ -38,7 +38,7 @@ cffi==1.14.6 # bcrypt # cryptography # pynacl -charset-normalizer==2.0.4 +charset-normalizer==2.0.6 # via requests click==7.1.2 # via @@ -50,9 +50,9 @@ croniter==1.0.15 # via flytekit cryptography==3.4.8 # via paramiko -dataclasses-json==0.5.5 +dataclasses-json==0.5.6 # via flytekit -decorator==5.0.9 +decorator==5.1.0 # via # ipython # retry @@ -73,13 +73,13 @@ entrypoints==0.3 # jupyter-client # nbconvert # papermill -flyteidl==0.21.0 +flyteidl==0.21.2 # via flytekit gevent==21.8.0 # via sagemaker-training -greenlet==1.1.1 +greenlet==1.1.2 # via gevent -grpcio==1.40.0 +grpcio==1.41.0 # via flytekit hmsclient==0.1.1 # via flytekit @@ -91,7 +91,7 @@ inotify_simple==1.2.1 # via sagemaker-training ipykernel==5.5.5 # via flytekit -ipython==7.27.0 +ipython==7.28.0 # via ipykernel ipython-genutils==0.2.0 # via nbformat @@ -105,11 +105,11 @@ jmespath==0.10.0 # botocore jsonschema==3.2.0 # via nbformat -jupyter-client==7.0.2 +jupyter-client==7.0.5 # via # ipykernel # nbclient -jupyter-core==4.7.1 +jupyter-core==4.8.1 # via # jupyter-client # nbconvert @@ -118,7 +118,7 @@ jupyterlab-pygments==0.1.2 # via nbconvert k8s-proto==0.0.3 # via flytekit -keyring==23.1.0 +keyring==23.2.1 # via flytekit markupsafe==2.0.1 # via jinja2 @@ -145,7 +145,7 @@ nbclient==0.5.4 # via # nbconvert # papermill -nbconvert==6.1.0 +nbconvert==6.2.0 # via flytekit nbformat==5.1.3 # via @@ -165,9 +165,9 @@ numpy==1.21.2 # scipy packaging==21.0 # via bleach -pandas==1.3.2 +pandas==1.3.3 # via flytekit -pandocfilters==1.4.3 +pandocfilters==1.5.0 # via nbconvert papermill==2.3.3 # via flytekit @@ -183,11 +183,11 @@ pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -platformdirs==2.3.0 +platformdirs==2.4.0 # via black prompt-toolkit==3.0.20 # via ipython -protobuf==3.17.3 +protobuf==3.18.0 # via # flyteidl # flytekit @@ -235,9 +235,9 @@ pytz==2018.4 # pandas pyyaml==5.4.1 # via papermill -pyzmq==22.2.1 +pyzmq==22.3.0 # via jupyter-client -regex==2021.8.28 +regex==2021.9.24 # via # black # docker-image-py @@ -246,7 +246,7 @@ requests==2.26.0 # flytekit # papermill # responses -responses==0.13.4 +responses==0.14.0 # via flytekit retry==0.9.2 # via flytekit @@ -267,7 +267,6 @@ six==1.16.0 # flytekit # grpcio # jsonschema - # protobuf # pynacl # python-dateutil # responses @@ -285,7 +284,7 @@ testpath==0.5.0 # via nbconvert textwrap3==0.9.2 # via ansiwrap -thrift==0.13.0 +thrift==0.15.0 # via hmsclient tomli==1.2.1 # via black @@ -293,7 +292,7 @@ tornado==6.1 # via # ipykernel # jupyter-client -tqdm==4.62.2 +tqdm==4.62.3 # via papermill traitlets==5.1.0 # via @@ -311,7 +310,7 @@ typing-extensions==3.10.0.2 # typing-inspect typing-inspect==0.7.1 # via dataclasses-json -urllib3==1.26.6 +urllib3==1.26.7 # via # botocore # flytekit @@ -329,7 +328,7 @@ wrapt==1.12.1 # via # deprecated # flytekit -zipp==3.5.0 +zipp==3.6.0 # via importlib-metadata zope.event==4.5.0 # via gevent diff --git a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/basic/joblib.py b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/basic/joblib.py new file mode 100644 index 0000000000..1f0e09e69a --- /dev/null +++ b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/basic/joblib.py @@ -0,0 +1,23 @@ +"""Test joblib file.""" + +import os +import typing + +import joblib + +import flytekit +from flytekit import task, workflow +from flytekit.types.file import JoblibSerializedFile + + +@task +def joblib_task(obj: typing.List[int]) -> JoblibSerializedFile: + working_dir = flytekit.current_context().working_directory + filename = os.path.join(working_dir, "object.joblib") + joblib.dump(obj, filename) + return JoblibSerializedFile(path=filename) + + +@workflow +def joblib_workflow(obj: typing.List[int]) -> JoblibSerializedFile: + return joblib_task(obj=obj) diff --git a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in index f7d015b843..8a4e3c9353 100644 --- a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in +++ b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in @@ -1,4 +1,5 @@ flytekit>=0.17.0b0 +joblib wheel matplotlib opencv-python diff --git a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt index af545501da..fa7f7f1806 100644 --- a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt +++ b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt @@ -2,13 +2,13 @@ # This file is autogenerated by pip-compile with python 3.8 # To update, run: # -# /Library/Developer/CommandLineTools/usr/bin/make requirements.txt +# make tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt # attrs==21.2.0 # via scantree certifi==2021.5.30 # via requests -chardet==4.0.0 +charset-normalizer==2.0.6 # via requests click==7.1.2 # via flytekit @@ -16,31 +16,37 @@ croniter==1.0.15 # via flytekit cycler==0.10.0 # via matplotlib -dataclasses-json==0.5.4 +dataclasses-json==0.5.6 # via flytekit -decorator==5.0.9 +decorator==5.1.0 # via retry -deprecated==1.2.12 +deprecated==1.2.13 # via flytekit dirhash==0.2.1 # via flytekit -docker-image-py==0.1.10 +diskcache==5.2.1 # via flytekit -flyteidl==0.19.11 +docker-image-py==0.1.12 # via flytekit -flytekit==0.20.1 - # via -r requirements.in -grpcio==1.38.1 +docstring-parser==0.10 # via flytekit -idna==2.10 +flyteidl==0.20.2 + # via flytekit +flytekit==0.22.2 + # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in +grpcio==1.41.0 + # via flytekit +idna==3.2 # via requests -importlib-metadata==4.6.1 +importlib-metadata==4.8.1 # via keyring -keyring==23.0.1 +joblib==1.0.1 + # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in +keyring==23.2.1 # via flytekit -kiwisolver==1.3.1 +kiwisolver==1.3.2 # via matplotlib -marshmallow==3.12.2 +marshmallow==3.13.0 # via # dataclasses-json # marshmallow-enum @@ -49,27 +55,27 @@ marshmallow-enum==1.5.1 # via dataclasses-json marshmallow-jsonschema==0.12.0 # via flytekit -matplotlib==3.4.2 - # via -r requirements.in +matplotlib==3.4.3 + # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in mypy-extensions==0.4.3 # via typing-inspect natsort==7.1.1 # via flytekit -numpy==1.21.0 +numpy==1.21.2 # via # matplotlib # opencv-python # pandas # pyarrow -opencv-python==4.5.2.54 - # via -r requirements.in -pandas==1.3.0 +opencv-python==4.5.3.56 + # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in +pandas==1.3.3 # via flytekit -pathspec==0.8.1 +pathspec==0.9.0 # via scantree -pillow==8.3.1 +pillow==8.3.2 # via matplotlib -protobuf==3.17.3 +protobuf==3.18.0 # via # flyteidl # flytekit @@ -85,7 +91,7 @@ python-dateutil==2.8.1 # flytekit # matplotlib # pandas -python-json-logger==2.0.1 +python-json-logger==2.0.2 # via flytekit pytimeparse==1.1.8 # via flytekit @@ -93,13 +99,13 @@ pytz==2018.4 # via # flytekit # pandas -regex==2021.7.6 +regex==2021.9.24 # via docker-image-py -requests==2.25.1 +requests==2.26.0 # via # flytekit # responses -responses==0.13.3 +responses==0.14.0 # via flytekit retry==0.9.2 # via flytekit @@ -110,7 +116,6 @@ six==1.16.0 # cycler # flytekit # grpcio - # protobuf # python-dateutil # responses # scantree @@ -118,24 +123,22 @@ sortedcontainers==2.4.0 # via flytekit statsd==3.3.0 # via flytekit -stringcase==1.2.0 - # via dataclasses-json -typing-extensions==3.10.0.0 +typing-extensions==3.10.0.2 # via typing-inspect typing-inspect==0.7.1 # via dataclasses-json -urllib3==1.26.6 +urllib3==1.26.7 # via # flytekit # requests # responses -wheel==0.36.2 +wheel==0.37.0 # via - # -r requirements.in + # -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in # flytekit wrapt==1.12.1 # via # deprecated # flytekit -zipp==3.5.0 +zipp==3.6.0 # via importlib-metadata diff --git a/tests/flytekit/integration/remote/test_remote.py b/tests/flytekit/integration/remote/test_remote.py index 70181e392a..d0af4933f3 100644 --- a/tests/flytekit/integration/remote/test_remote.py +++ b/tests/flytekit/integration/remote/test_remote.py @@ -5,6 +5,7 @@ import time import typing +import joblib import pytest from flytekit import kwtypes @@ -269,3 +270,15 @@ def test_execute_sqlite3_task(flyteclient, flyte_workflows_register, flyte_remot assert result.__class__.__name__ == "DataFrame" assert "TrackId" in result assert "Name" in result + + +def test_execute_joblib_workflow(flyteclient, flyte_workflows_register, flyte_remote_env): + remote = FlyteRemote.from_config(PROJECT, "development") + flyte_workflow = remote.fetch_workflow(name="workflows.basic.joblib.joblib_workflow", version=f"v{VERSION}") + input_obj = [1, 2, 3] + execution = remote.execute(flyte_workflow, {"obj": input_obj}, wait=True) + joblib_output = execution.outputs["o0"] + joblib_output.download() + output_obj = joblib.load(joblib_output.path) + assert execution.outputs["o0"].extension() == "joblib" + assert output_obj == input_obj