From 4693a20e4e8ff429ac9ea67959c9ed9441f00487 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Wed, 7 Dec 2022 14:31:59 -0800 Subject: [PATCH 1/8] Set flyteidl, grpcio, and grpcio-status new lower bounds Signed-off-by: Eduardo Apolinario --- setup.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 95e682cce9..94499f67eb 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ ] }, install_requires=[ - "flyteidl>=1.2.0,<1.3.0", + "flyteidl>=1.3.0,<1.4.0", "wheel>=0.30.0,<1.0.0", "pandas>=1.0.0,<2.0.0", "pyarrow>=4.0.0,<11.0.0", @@ -48,14 +48,13 @@ "deprecated>=1.0,<2.0", "docker>=5.0.3,<7.0.0", "python-dateutil>=2.1", - # Restrict grpcio and grpcio-status. The 1.49.1 version pulls in a version of protobuf that is not compatible. - # More details in https://github.com/flyteorg/flyte/issues/3006 - "grpcio>=1.43.0,!=1.45.0,<1.49.1,<2.0", - "grpcio-status>=1.43,!=1.45.0,<1.49.1", + # Restrict grpcio and grpcio-status. Version 1.50.0 pulls in a version of protobuf that is not compatible + # with the old protobuf library (as described in https://developers.google.com/protocol-buffers/docs/news/2022-05-06) + "grpcio>=1.50.0,<2.0", + "grpcio-status>=1.50.0,<2.0", "importlib-metadata", "pyopenssl", "joblib", - "protobuf>=3.6.1,<4", "python-json-logger>=2.0.0", "pytimeparse>=1.1.8,<2.0.0", "pytz", From a4fb57b0a10a278bdaf91b4d783205425684f354 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Wed, 7 Dec 2022 14:32:15 -0800 Subject: [PATCH 2/8] Regenerate requirements Signed-off-by: Eduardo Apolinario --- dev-requirements.txt | 60 ++++++++++--------- requirements-spark2.txt | 30 +++++----- requirements.txt | 36 ++++++----- .../workflows/requirements.txt | 22 +++---- 4 files changed, 78 insertions(+), 70 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 5a73546c0d..3438ae7a8e 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.7 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.7 +# by the following command: # # make dev-requirements.txt # @@ -8,8 +8,6 @@ # via # -c requirements.txt # pytest-flyte -appnope==0.1.3 - # via ipython arrow==1.2.3 # via # -c requirements.txt @@ -32,7 +30,7 @@ cached-property==1.5.2 # via docker-compose cachetools==5.2.0 # via google-auth -certifi==2022.9.24 +certifi==2022.12.7 # via # -c requirements.txt # requests @@ -43,7 +41,7 @@ cffi==1.15.1 # pynacl cfgv==3.3.1 # via pre-commit -chardet==5.0.0 +chardet==5.1.0 # via # -c requirements.txt # binaryornot @@ -70,15 +68,16 @@ coverage[toml]==6.5.0 # via # -r dev-requirements.in # pytest-cov -croniter==1.3.7 +croniter==1.3.8 # via # -c requirements.txt # flytekit -cryptography==38.0.3 +cryptography==38.0.4 # via # -c requirements.txt # paramiko # pyopenssl + # secretstorage dataclasses-json==0.5.7 # via # -c requirements.txt @@ -121,18 +120,18 @@ docstring-parser==0.15 # flytekit exceptiongroup==1.0.4 # via pytest -filelock==3.8.0 +filelock==3.8.2 # via virtualenv -flyteidl==1.2.5 +flyteidl==1.3.0 # via # -c requirements.txt # flytekit -google-api-core[grpc]==2.10.2 +google-api-core[grpc]==2.11.0 # via # google-cloud-bigquery # google-cloud-bigquery-storage # google-cloud-core -google-auth==2.14.1 +google-auth==2.15.0 # via # google-api-core # google-cloud-core @@ -154,14 +153,14 @@ googleapis-common-protos==1.57.0 # flyteidl # google-api-core # grpcio-status -grpcio==1.48.2 +grpcio==1.51.1 # via # -c requirements.txt # flytekit # google-api-core # google-cloud-bigquery # grpcio-status -grpcio-status==1.48.2 +grpcio-status==1.51.1 # via # -c requirements.txt # flytekit @@ -172,7 +171,7 @@ idna==3.4 # via # -c requirements.txt # requests -importlib-metadata==5.0.0 +importlib-metadata==5.1.0 # via # -c requirements.txt # click @@ -191,8 +190,13 @@ jaraco-classes==3.2.3 # via # -c requirements.txt # keyring -jedi==0.18.1 +jedi==0.18.2 # via ipython +jeepney==0.8.0 + # via + # -c requirements.txt + # keyring + # secretstorage jinja2==3.1.2 # via # -c requirements.txt @@ -283,23 +287,22 @@ pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -platformdirs==2.5.4 +platformdirs==2.6.0 # via virtualenv pluggy==1.0.0 # via pytest pre-commit==2.20.0 # via -r dev-requirements.in -prompt-toolkit==3.0.32 +prompt-toolkit==3.0.36 # via ipython proto-plus==1.22.1 # via # google-cloud-bigquery # google-cloud-bigquery-storage -protobuf==3.20.3 +protobuf==4.21.10 # via # -c requirements.txt # flyteidl - # flytekit # google-api-core # google-cloud-bigquery # google-cloud-bigquery-storage @@ -317,7 +320,7 @@ py==1.11.0 # via # -c requirements.txt # retry -pyarrow==10.0.0 +pyarrow==10.0.1 # via # -c requirements.txt # flytekit @@ -422,6 +425,10 @@ scikit-learn==1.0.2 # via -r dev-requirements.in scipy==1.7.3 # via scikit-learn +secretstorage==3.3.3 + # via + # -c requirements.txt + # keyring singledispatchmethod==1.0 # via # -c requirements.txt @@ -431,7 +438,6 @@ six==1.16.0 # -c requirements.txt # dockerpty # google-auth - # grpcio # jsonschema # paramiko # python-dateutil @@ -448,7 +454,7 @@ text-unidecode==1.3 # via # -c requirements.txt # python-slugify -texttable==1.6.4 +texttable==1.6.7 # via docker-compose threadpoolctl==3.1.0 # via scikit-learn @@ -464,7 +470,7 @@ tomli==2.0.1 # pytest torch==1.12.1 # via -r dev-requirements.in -traitlets==5.5.0 +traitlets==5.6.0 # via # ipython # matplotlib-inline @@ -488,14 +494,14 @@ typing-inspect==0.8.0 # via # -c requirements.txt # dataclasses-json -urllib3==1.26.12 +urllib3==1.26.13 # via # -c requirements.txt # docker # flytekit # requests # responses -virtualenv==20.16.7 +virtualenv==20.17.1 # via pre-commit wcwidth==0.2.5 # via prompt-toolkit @@ -513,7 +519,7 @@ wrapt==1.14.1 # -c requirements.txt # deprecated # flytekit -zipp==3.10.0 +zipp==3.11.0 # via # -c requirements.txt # importlib-metadata diff --git a/requirements-spark2.txt b/requirements-spark2.txt index 783282f409..3c07cb3cee 100644 --- a/requirements-spark2.txt +++ b/requirements-spark2.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.7 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.7 +# by the following command: # # make requirements-spark2.txt # @@ -16,11 +16,11 @@ attrs==20.3.0 # jsonschema binaryornot==0.4.4 # via cookiecutter -certifi==2022.9.24 +certifi==2022.12.7 # via requests cffi==1.15.1 # via cryptography -chardet==5.0.0 +chardet==5.1.0 # via binaryornot charset-normalizer==2.1.1 # via requests @@ -32,9 +32,9 @@ cloudpickle==2.2.0 # via flytekit cookiecutter==2.1.1 # via flytekit -croniter==1.3.7 +croniter==1.3.8 # via flytekit -cryptography==38.0.3 +cryptography==38.0.4 # via # pyopenssl # secretstorage @@ -52,21 +52,21 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.15 # via flytekit -flyteidl==1.2.5 +flyteidl==1.3.0 # via flytekit googleapis-common-protos==1.57.0 # via # flyteidl # grpcio-status -grpcio==1.48.2 +grpcio==1.51.1 # via # flytekit # grpcio-status -grpcio-status==1.48.2 +grpcio-status==1.51.1 # via flytekit idna==3.4 # via requests -importlib-metadata==5.0.0 +importlib-metadata==5.1.0 # via # click # flytekit @@ -121,10 +121,9 @@ pandas==1.3.5 # via # -r requirements.in # flytekit -protobuf==3.20.3 +protobuf==4.21.10 # via # flyteidl - # flytekit # googleapis-common-protos # grpcio-status # protoc-gen-swagger @@ -132,7 +131,7 @@ protoc-gen-swagger==0.1.0 # via flyteidl py==1.11.0 # via retry -pyarrow==10.0.0 +pyarrow==10.0.1 # via flytekit pycparser==2.21 # via cffi @@ -181,7 +180,6 @@ singledispatchmethod==1.0 # via flytekit six==1.16.0 # via - # grpcio # jsonschema # python-dateutil # websocket-client @@ -204,7 +202,7 @@ typing-extensions==4.4.0 # typing-inspect typing-inspect==0.8.0 # via dataclasses-json -urllib3==1.26.12 +urllib3==1.26.13 # via # docker # flytekit @@ -220,7 +218,7 @@ wrapt==1.14.1 # via # deprecated # flytekit -zipp==3.10.0 +zipp==3.11.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements.txt b/requirements.txt index e4d3fbcdc0..bb994fb958 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.7 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.7 +# by the following command: # # make requirements.txt # @@ -14,11 +14,11 @@ attrs==20.3.0 # jsonschema binaryornot==0.4.4 # via cookiecutter -certifi==2022.9.24 +certifi==2022.12.7 # via requests cffi==1.15.1 # via cryptography -chardet==5.0.0 +chardet==5.1.0 # via binaryornot charset-normalizer==2.1.1 # via requests @@ -30,9 +30,9 @@ cloudpickle==2.2.0 # via flytekit cookiecutter==2.1.1 # via flytekit -croniter==1.3.7 +croniter==1.3.8 # via flytekit -cryptography==38.0.3 +cryptography==38.0.4 # via # pyopenssl # secretstorage @@ -50,21 +50,21 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.15 # via flytekit -flyteidl==1.2.5 +flyteidl==1.3.0 # via flytekit googleapis-common-protos==1.57.0 # via # flyteidl # grpcio-status -grpcio==1.48.2 +grpcio==1.51.1 # via # flytekit # grpcio-status -grpcio-status==1.48.2 +grpcio-status==1.51.1 # via flytekit idna==3.4 # via requests -importlib-metadata==5.0.0 +importlib-metadata==5.1.0 # via # click # flytekit @@ -72,6 +72,10 @@ importlib-metadata==5.0.0 # keyring jaraco-classes==3.2.3 # via keyring +jeepney==0.8.0 + # via + # keyring + # secretstorage jinja2==3.1.2 # via # cookiecutter @@ -115,10 +119,9 @@ pandas==1.3.5 # via # -r requirements.in # flytekit -protobuf==3.20.3 +protobuf==4.21.10 # via # flyteidl - # flytekit # googleapis-common-protos # grpcio-status # protoc-gen-swagger @@ -126,7 +129,7 @@ protoc-gen-swagger==0.1.0 # via flyteidl py==1.11.0 # via retry -pyarrow==10.0.0 +pyarrow==10.0.1 # via flytekit pycparser==2.21 # via cffi @@ -169,11 +172,12 @@ responses==0.22.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.3 + # via keyring singledispatchmethod==1.0 # via flytekit six==1.16.0 # via - # grpcio # jsonschema # python-dateutil # websocket-client @@ -196,7 +200,7 @@ typing-extensions==4.4.0 # typing-inspect typing-inspect==0.8.0 # via dataclasses-json -urllib3==1.26.12 +urllib3==1.26.13 # via # docker # flytekit @@ -212,7 +216,7 @@ wrapt==1.14.1 # via # deprecated # flytekit -zipp==3.10.0 +zipp==3.11.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt index 71225babcf..68abcac8e0 100644 --- a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt +++ b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.7 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.7 +# by the following command: # # make tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt # @@ -8,11 +8,11 @@ arrow==1.2.3 # via jinja2-time binaryornot==0.4.4 # via cookiecutter -certifi==2022.9.24 +certifi==2022.12.7 # via requests cffi==1.15.1 # via cryptography -chardet==5.0.0 +chardet==5.1.0 # via binaryornot charset-normalizer==2.1.1 # via requests @@ -24,9 +24,9 @@ cloudpickle==2.2.0 # via flytekit cookiecutter==2.1.1 # via flytekit -croniter==1.3.7 +croniter==1.3.8 # via flytekit -cryptography==38.0.3 +cryptography==38.0.4 # via # pyopenssl # secretstorage @@ -48,7 +48,7 @@ docstring-parser==0.15 # via flytekit flyteidl==1.2.5 # via flytekit -flytekit==1.2.4 +flytekit==1.2.5 # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in fonttools==4.38.0 # via matplotlib @@ -64,7 +64,7 @@ grpcio-status==1.48.2 # via flytekit idna==3.4 # via requests -importlib-metadata==5.0.0 +importlib-metadata==5.1.0 # via # click # flytekit @@ -137,7 +137,7 @@ protoc-gen-swagger==0.1.0 # via flyteidl py==1.11.0 # via retry -pyarrow==6.0.1 +pyarrow==10.0.1 # via flytekit pycparser==2.21 # via cffi @@ -208,7 +208,7 @@ typing-extensions==4.4.0 # typing-inspect typing-inspect==0.8.0 # via dataclasses-json -urllib3==1.26.12 +urllib3==1.26.13 # via # docker # flytekit @@ -224,5 +224,5 @@ wrapt==1.14.1 # via # deprecated # flytekit -zipp==3.10.0 +zipp==3.11.0 # via importlib-metadata From e1bfe5809b2129cd675425123eec3b89677d7660 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Wed, 7 Dec 2022 22:27:38 -0800 Subject: [PATCH 3/8] Single out tensorflow tests and fix protobuf type transformer tests Signed-off-by: Eduardo Apolinario --- Makefile | 5 ++++- flytekit/__init__.py | 1 - flytekit/core/type_engine.py | 6 +++--- flytekit/extras/tensorflow/__init__.py | 3 +++ 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 4b3278bec0..2f71c97ebe 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,10 @@ test: lint unit_test .PHONY: unit_test unit_test: - pytest -m "not sandbox_test" tests/flytekit/unit + # Skip tensorflow tests and run them with the necessary env var set so that a working (albeit slower) + # library is used to serialize/deserialize protobufs is used. + pytest -m "not sandbox_test" tests/flytekit/unit/ --ignore=tests/flytekit/unit/extras/tensorflow && \ + PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python pytest tests/flytekit/unit/extras/tensorflow requirements-spark2.txt: export CUSTOM_COMPILE_COMMAND := make requirements-spark2.txt requirements-spark2.txt: requirements-spark2.in install-piptools diff --git a/flytekit/__init__.py b/flytekit/__init__.py index a18d476291..9a2ed7f180 100644 --- a/flytekit/__init__.py +++ b/flytekit/__init__.py @@ -154,7 +154,6 @@ LiteralType BlobType """ - import sys from typing import Generator diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index 98969e41b3..35e2a3e39d 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -15,10 +15,10 @@ from dataclasses_json import DataClassJsonMixin, dataclass_json from google.protobuf import json_format as _json_format -from google.protobuf import reflection as _proto_reflection from google.protobuf import struct_pb2 as _struct from google.protobuf.json_format import MessageToDict as _MessageToDict from google.protobuf.json_format import ParseDict as _ParseDict +from google.protobuf.message import Message from google.protobuf.struct_pb2 import Struct from marshmallow_enum import EnumField, LoadDumpOptions from marshmallow_jsonschema import JSONSchema @@ -551,11 +551,11 @@ def guess_python_type(self, literal_type: LiteralType) -> Type[T]: raise ValueError(f"Dataclass transformer cannot reverse {literal_type}") -class ProtobufTransformer(TypeTransformer[_proto_reflection.GeneratedProtocolMessageType]): +class ProtobufTransformer(TypeTransformer[Message]): PB_FIELD_KEY = "pb_type" def __init__(self): - super().__init__("Protobuf-Transformer", _proto_reflection.GeneratedProtocolMessageType) + super().__init__("Protobuf-Transformer", Message) @staticmethod def tag(expected_python_type: Type[T]) -> str: diff --git a/flytekit/extras/tensorflow/__init__.py b/flytekit/extras/tensorflow/__init__.py index c42b1fd3d5..f51da24dae 100644 --- a/flytekit/extras/tensorflow/__init__.py +++ b/flytekit/extras/tensorflow/__init__.py @@ -18,6 +18,9 @@ import tensorflow _tensorflow_installed = True +except TypeError as e: + logger.warn(f"Unsupported version of tensorflow installed. Error message from protobuf library: {e}") + _tensorflow_installed = False except (ImportError, OSError): _tensorflow_installed = False From d03422452f5000450cdb8e10129fa4ada5fe297d Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Wed, 7 Dec 2022 23:44:50 -0800 Subject: [PATCH 4/8] Comment out skl2onnx and tf2oonx in docs Signed-off-by: Eduardo Apolinario --- doc-requirements.in | 15 ++- doc-requirements.txt | 239 ++++++++++++++++++++++--------------------- 2 files changed, 132 insertions(+), 122 deletions(-) diff --git a/doc-requirements.in b/doc-requirements.in index d61955fbff..7a1c95353c 100644 --- a/doc-requirements.in +++ b/doc-requirements.in @@ -12,11 +12,9 @@ sphinx-copybutton sphinx_fontawesome sphinx-panels sphinxcontrib-yt -grpcio==1.43.0 -grpcio-status==1.43.0 cryptography -google-api-core[grpc]==2.8.2 -scikit-learn==1.1.1 +google-api-core[grpc] +scikit-learn sphinx-tags # Packages for Plugin docs @@ -24,7 +22,7 @@ sphinx-tags botocore # fsspec fsspec # fsspec google-cloud # bigquery -google-cloud-bigquery==3.1.0 # bigquery +google-cloud-bigquery # bigquery markdown # deck plotly # deck pandas_profiling # deck @@ -38,9 +36,10 @@ jupyter # papermill pyspark # spark sqlalchemy # sqlalchemy torch # pytorch -skl2onnx # onnxscikitlearn -tf2onnx # onnxtensorflow -tensorflow==2.9.0 # onnxtensorflow +# TODO: Remove after buf migration is done and packages updated +# skl2onnx # onnxscikitlearn +# tf2onnx # onnxtensorflow +tensorflow==2.8.1 # onnxtensorflow whylogs # whylogs whylabs-client # whylogs ray # ray diff --git a/doc-requirements.txt b/doc-requirements.txt index 4991bd4adc..fefdd3c2da 100644 --- a/doc-requirements.txt +++ b/doc-requirements.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: # # make doc-requirements.txt # @@ -20,10 +20,6 @@ ansiwrap==0.8.4 # via papermill anyio==3.6.2 # via jupyter-server -appnope==0.1.3 - # via - # ipykernel - # ipython argon2-cffi==21.3.0 # via # jupyter-server @@ -32,10 +28,12 @@ argon2-cffi==21.3.0 argon2-cffi-bindings==21.2.0 # via argon2-cffi arrow==1.2.3 - # via jinja2-time -astroid==2.12.12 + # via + # isoduration + # jinja2-time +astroid==2.12.13 # via sphinx-autoapi -asttokens==2.1.0 +asttokens==2.2.1 # via stack-data astunparse==1.6.3 # via tensorflow @@ -58,11 +56,11 @@ binaryornot==0.4.4 # via cookiecutter bleach==5.0.1 # via nbconvert -botocore==1.29.10 +botocore==1.29.25 # via -r doc-requirements.in cachetools==5.2.0 # via google-auth -certifi==2022.9.24 +certifi==2022.12.7 # via # kubernetes # requests @@ -72,7 +70,7 @@ cffi==1.15.1 # cryptography cfgv==3.3.1 # via pre-commit -chardet==5.0.0 +chardet==5.1.0 # via binaryornot charset-normalizer==2.1.1 # via requests @@ -87,15 +85,20 @@ cloudpickle==2.2.0 # via flytekit colorama==0.4.6 # via great-expectations +comm==0.1.1 + # via ipykernel +contourpy==1.0.6 + # via matplotlib cookiecutter==2.1.1 # via flytekit -croniter==1.3.7 +croniter==1.3.8 # via flytekit -cryptography==38.0.3 +cryptography==38.0.4 # via # -r doc-requirements.in # great-expectations # pyopenssl + # secretstorage css-html-js-minify==2.5.5 # via sphinx-material cycler==0.11.0 @@ -104,7 +107,7 @@ dataclasses-json==0.5.7 # via # dolt-integrations # flytekit -debugpy==1.6.3 +debugpy==1.6.4 # via ipykernel decorator==5.1.1 # via @@ -141,18 +144,18 @@ executing==1.2.0 # via stack-data fastjsonschema==2.16.2 # via nbformat -filelock==3.8.0 +filelock==3.8.2 # via # ray # virtualenv -flatbuffers==1.12 - # via - # tensorflow - # tf2onnx -flyteidl==1.2.5 +flatbuffers==22.12.6 + # via tensorflow +flyteidl==1.3.0 # via flytekit fonttools==4.38.0 # via matplotlib +fqdn==1.5.1 + # via jsonschema frozenlist==1.3.3 # via # aiosignal @@ -163,15 +166,15 @@ fsspec==2022.11.0 # modin furo @ git+https://github.com/flyteorg/furo@main # via -r doc-requirements.in -gast==0.4.0 +gast==0.5.3 # via tensorflow -google-api-core[grpc]==2.8.2 +google-api-core[grpc]==2.11.0 # via # -r doc-requirements.in # google-cloud-bigquery # google-cloud-bigquery-storage # google-cloud-core -google-auth==2.14.1 +google-auth==2.15.0 # via # google-api-core # google-auth-oauthlib @@ -182,7 +185,7 @@ google-auth-oauthlib==0.4.6 # via tensorboard google-cloud==0.34.0 # via -r doc-requirements.in -google-cloud-bigquery==3.1.0 +google-cloud-bigquery==3.4.0 # via -r doc-requirements.in google-cloud-bigquery-storage==2.16.2 # via google-cloud-bigquery @@ -199,13 +202,12 @@ googleapis-common-protos==1.57.0 # flyteidl # google-api-core # grpcio-status -great-expectations==0.15.32 +great-expectations==0.15.36 # via -r doc-requirements.in greenlet==2.0.1 # via sqlalchemy -grpcio==1.43.0 +grpcio==1.51.1 # via - # -r doc-requirements.in # flytekit # google-api-core # google-cloud-bigquery @@ -213,26 +215,26 @@ grpcio==1.43.0 # ray # tensorboard # tensorflow -grpcio-status==1.43.0 +grpcio-status==1.51.1 # via - # -r doc-requirements.in # flytekit # google-api-core h5py==3.7.0 # via tensorflow htmlmin==0.1.12 # via pandas-profiling -identify==2.5.8 +identify==2.5.9 # via pre-commit idna==3.4 # via # anyio + # jsonschema # requests imagehash==4.3.1 # via visions imagesize==1.4.1 # via sphinx -importlib-metadata==5.0.0 +importlib-metadata==5.1.0 # via # flytekit # great-expectations @@ -240,7 +242,7 @@ importlib-metadata==5.0.0 # markdown # nbconvert # sphinx -ipykernel==6.17.1 +ipykernel==6.19.0 # via # ipywidgets # jupyter @@ -248,7 +250,7 @@ ipykernel==6.17.1 # nbclassic # notebook # qtconsole -ipython==8.6.0 +ipython==8.7.0 # via # great-expectations # ipykernel @@ -259,14 +261,20 @@ ipython-genutils==0.2.0 # nbclassic # notebook # qtconsole -ipywidgets==8.0.2 +ipywidgets==8.0.3 # via # great-expectations # jupyter +isoduration==20.11.0 + # via jsonschema jaraco-classes==3.2.3 # via keyring -jedi==0.18.1 +jedi==0.18.2 # via ipython +jeepney==0.8.0 + # via + # keyring + # secretstorage jinja2==3.1.2 # via # altair @@ -292,16 +300,19 @@ joblib==1.2.0 jsonpatch==1.32 # via great-expectations jsonpointer==2.3 - # via jsonpatch -jsonschema==4.7.2 + # via + # jsonpatch + # jsonschema +jsonschema[format-nongpl]==4.7.2 # via # altair # great-expectations + # jupyter-events # nbformat # ray jupyter==1.0.0 # via -r doc-requirements.in -jupyter-client==7.4.6 +jupyter-client==7.4.8 # via # ipykernel # jupyter-console @@ -312,24 +323,29 @@ jupyter-client==7.4.6 # qtconsole jupyter-console==6.4.4 # via jupyter -jupyter-core==5.0.0 +jupyter-core==5.1.0 # via # jupyter-client # jupyter-server # nbclassic + # nbclient # nbconvert # nbformat # notebook # qtconsole -jupyter-server==1.23.2 +jupyter-events==0.5.0 + # via jupyter-server +jupyter-server==2.0.0 # via # nbclassic # notebook-shim +jupyter-server-terminals==0.4.2 + # via jupyter-server jupyterlab-pygments==0.2.2 # via nbconvert -jupyterlab-widgets==3.0.3 +jupyterlab-widgets==3.0.4 # via ipywidgets -keras==2.9.0 +keras==2.8.0 # via tensorflow keras-preprocessing==1.1.2 # via tensorflow @@ -366,9 +382,8 @@ marshmallow-enum==1.5.1 # via dataclasses-json marshmallow-jsonschema==0.13.0 # via flytekit -matplotlib==3.5.3 +matplotlib==3.6.2 # via - # missingno # pandas-profiling # phik # seaborn @@ -376,13 +391,11 @@ matplotlib-inline==0.1.6 # via # ipykernel # ipython -missingno==0.5.1 - # via pandas-profiling mistune==2.0.4 # via # great-expectations # nbconvert -modin==0.17.0 +modin==0.17.1 # via -r doc-requirements.in more-itertools==9.0.0 # via jaraco-classes @@ -398,11 +411,11 @@ natsort==8.2.0 # via flytekit nbclassic==0.4.8 # via notebook -nbclient==0.7.0 +nbclient==0.7.2 # via # nbconvert # papermill -nbconvert==7.2.5 +nbconvert==7.2.6 # via # jupyter # jupyter-server @@ -422,7 +435,6 @@ nest-asyncio==1.5.6 # ipykernel # jupyter-client # nbclassic - # nbclient # notebook networkx==2.8.8 # via visions @@ -434,18 +446,16 @@ notebook==6.5.2 # jupyter notebook-shim==0.2.2 # via nbclassic -numpy==1.23.4 +numpy==1.23.5 # via # altair + # contourpy # great-expectations # h5py # imagehash # keras-preprocessing # matplotlib - # missingno # modin - # onnx - # onnxconverter-common # opt-einsum # pandas # pandas-profiling @@ -458,21 +468,22 @@ numpy==1.23.4 # scikit-learn # scipy # seaborn - # skl2onnx # statsmodels # tensorboard # tensorflow - # tf2onnx # visions +nvidia-cublas-cu11==11.10.3.66 + # via + # nvidia-cudnn-cu11 + # torch +nvidia-cuda-nvrtc-cu11==11.7.99 + # via torch +nvidia-cuda-runtime-cu11==11.7.99 + # via torch +nvidia-cudnn-cu11==8.5.0.96 + # via torch oauthlib==3.2.2 # via requests-oauthlib -onnx==1.12.0 - # via - # onnxconverter-common - # skl2onnx - # tf2onnx -onnxconverter-common==1.13.0 - # via skl2onnx opt-einsum==3.3.0 # via tensorflow packaging==21.3 @@ -486,13 +497,11 @@ packaging==21.3 # matplotlib # modin # nbconvert - # onnxconverter-common # pandera # qtpy # sphinx # statsmodels - # tensorflow -pandas==1.5.1 +pandas==1.5.2 # via # altair # dolt-integrations @@ -505,7 +514,7 @@ pandas==1.5.1 # seaborn # statsmodels # visions -pandas-profiling==3.4.0 +pandas-profiling==3.5.0 # via -r doc-requirements.in pandera==0.13.4 # via -r doc-requirements.in @@ -519,7 +528,7 @@ patsy==0.5.3 # via statsmodels pexpect==4.8.0 # via ipython -phik==0.12.2 +phik==0.12.3 # via pandas-profiling pickleshare==0.7.5 # via ipython @@ -528,7 +537,7 @@ pillow==9.3.0 # imagehash # matplotlib # visions -platformdirs==2.5.4 +platformdirs==2.6.0 # via # jupyter-core # virtualenv @@ -541,7 +550,7 @@ prometheus-client==0.15.0 # jupyter-server # nbclassic # notebook -prompt-toolkit==3.0.32 +prompt-toolkit==3.0.36 # via # ipython # jupyter-console @@ -549,21 +558,17 @@ proto-plus==1.22.1 # via # google-cloud-bigquery # google-cloud-bigquery-storage -protobuf==3.19.6 +protobuf==4.21.10 # via # flyteidl - # flytekit # google-api-core # google-cloud-bigquery # google-cloud-bigquery-storage # googleapis-common-protos # grpcio-status - # onnx - # onnxconverter-common # proto-plus # protoc-gen-swagger # ray - # skl2onnx # tensorboard # tensorflow # whylogs @@ -583,7 +588,7 @@ py==1.11.0 # via retry py4j==0.10.9.5 # via pyspark -pyarrow==6.0.1 +pyarrow==10.0.1 # via # flytekit # google-cloud-bigquery @@ -597,6 +602,7 @@ pycparser==2.21 # via cffi pydantic==1.10.2 # via + # great-expectations # pandas-profiling # pandera pygments==2.13.0 @@ -633,8 +639,10 @@ python-dateutil==2.8.2 # pandas # whylabs-client python-json-logger==2.0.4 - # via flytekit -python-slugify[unidecode]==6.1.2 + # via + # flytekit + # jupyter-events +python-slugify[unidecode]==7.0.0 # via # cookiecutter # sphinx-material @@ -654,6 +662,7 @@ pyyaml==6.0 # via # cookiecutter # flytekit + # jupyter-events # kubernetes # pandas-profiling # papermill @@ -692,7 +701,6 @@ requests==2.28.1 # responses # sphinx # tensorboard - # tf2onnx requests-oauthlib==1.3.1 # via # google-auth-oauthlib @@ -701,30 +709,30 @@ responses==0.22.0 # via flytekit retry==0.9.2 # via flytekit +rfc3339-validator==0.1.4 + # via jsonschema +rfc3986-validator==0.1.1 + # via jsonschema rsa==4.9 # via google-auth ruamel-yaml==0.17.17 # via great-expectations ruamel-yaml-clib==0.2.7 # via ruamel-yaml -scikit-learn==1.1.1 - # via - # -r doc-requirements.in - # skl2onnx +scikit-learn==1.1.3 + # via -r doc-requirements.in scipy==1.9.3 # via # great-expectations # imagehash - # missingno # pandas-profiling # phik # scikit-learn - # skl2onnx # statsmodels seaborn==0.12.1 - # via - # missingno - # pandas-profiling + # via pandas-profiling +secretstorage==3.3.3 + # via keyring send2trash==1.8.0 # via # jupyter-server @@ -737,16 +745,13 @@ six==1.16.0 # bleach # google-auth # google-pasta - # grpcio # keras-preprocessing # kubernetes # patsy # python-dateutil + # rfc3339-validator # sphinx-code-include # tensorflow - # tf2onnx -skl2onnx==1.13 - # via -r doc-requirements.in sniffio==1.3.0 # via anyio snowballstemmer==2.2.0 @@ -806,7 +811,7 @@ sphinxcontrib-yt==0.2.2 # via -r doc-requirements.in sqlalchemy==1.4.44 # via -r doc-requirements.in -stack-data==0.6.1 +stack-data==0.6.2 # via ipython statsd==3.3.0 # via flytekit @@ -818,33 +823,30 @@ tenacity==8.1.0 # via # papermill # plotly -tensorboard==2.9.1 +tensorboard==2.8.0 # via tensorflow tensorboard-data-server==0.6.1 # via tensorboard tensorboard-plugin-wit==1.8.1 # via tensorboard -tensorflow==2.9.0 +tensorflow==2.8.1 # via -r doc-requirements.in -tensorflow-estimator==2.9.0 +tensorflow-estimator==2.8.0 # via tensorflow -tensorflow-io-gcs-filesystem==0.27.0 +tensorflow-io-gcs-filesystem==0.28.0 # via tensorflow -termcolor==2.0.1 - # via - # great-expectations - # tensorflow -terminado==0.17.0 +termcolor==2.1.1 + # via tensorflow +terminado==0.17.1 # via # jupyter-server + # jupyter-server-terminals # nbclassic # notebook text-unidecode==1.3 # via python-slugify textwrap3==0.9.2 # via ansiwrap -tf2onnx==1.13.0 - # via -r doc-requirements.in threadpoolctl==3.1.0 # via scikit-learn tinycss2==1.2.1 @@ -870,13 +872,15 @@ tqdm==4.64.1 # great-expectations # pandas-profiling # papermill -traitlets==5.5.0 +traitlets==5.6.0 # via + # comm # ipykernel # ipython # ipywidgets # jupyter-client # jupyter-core + # jupyter-events # jupyter-server # matplotlib-inline # nbclassic @@ -885,6 +889,8 @@ traitlets==5.5.0 # nbformat # notebook # qtconsole +typeguard==2.13.3 + # via pandas-profiling types-toml==0.10.8.1 # via responses typing-extensions==4.4.0 @@ -892,7 +898,6 @@ typing-extensions==4.4.0 # astroid # flytekit # great-expectations - # onnx # pydantic # tensorflow # torch @@ -902,7 +907,7 @@ typing-inspect==0.8.0 # via # dataclasses-json # pandera -tzdata==2022.6 +tzdata==2022.7 # via pytz-deprecation-shim tzlocal==4.2 # via great-expectations @@ -910,7 +915,9 @@ unidecode==1.3.6 # via # python-slugify # sphinx-autoapi -urllib3==1.26.12 +uri-template==1.2.0 + # via jsonschema +urllib3==1.26.13 # via # botocore # docker @@ -920,7 +927,7 @@ urllib3==1.26.12 # requests # responses # whylabs-client -virtualenv==20.16.7 +virtualenv==20.17.1 # via # pre-commit # ray @@ -928,6 +935,8 @@ visions[type_image_path]==0.7.5 # via pandas-profiling wcwidth==0.2.5 # via prompt-toolkit +webcolors==1.12 + # via jsonschema webencodings==0.5.1 # via # bleach @@ -943,14 +952,16 @@ wheel==0.38.4 # via # astunparse # flytekit + # nvidia-cublas-cu11 + # nvidia-cuda-runtime-cu11 # tensorboard -whylabs-client==0.4.0 +whylabs-client==0.4.2 # via -r doc-requirements.in -whylogs==1.1.13 +whylogs==1.1.16 # via -r doc-requirements.in whylogs-sketching==3.4.1.dev3 # via whylogs -widgetsnbextension==4.0.3 +widgetsnbextension==4.0.4 # via ipywidgets wrapt==1.14.1 # via @@ -959,7 +970,7 @@ wrapt==1.14.1 # flytekit # pandera # tensorflow -zipp==3.10.0 +zipp==3.11.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: From 6217fde49cde8537d91b2ce47bac613ab8dd83ab Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Wed, 7 Dec 2022 23:45:17 -0800 Subject: [PATCH 5/8] More requirements and dev-requirements shenanigans. Signed-off-by: Eduardo Apolinario --- dev-requirements.in | 3 +-- dev-requirements.txt | 35 ++++++++++++++++++++++++++++++----- requirements.in | 2 ++ requirements.txt | 1 + 4 files changed, 34 insertions(+), 7 deletions(-) diff --git a/dev-requirements.in b/dev-requirements.in index c2a0a9bdd5..a02c8fa144 100644 --- a/dev-requirements.in +++ b/dev-requirements.in @@ -12,8 +12,7 @@ codespell google-cloud-bigquery google-cloud-bigquery-storage IPython -tensorflow -grpcio-status<1.49.0 +tensorflow==2.8.1 # Newer versions of torch bring in nvidia dependencies that are not present in windows, so # we put this constraint while we do not have per-environment requirements files torch<=1.12.1 diff --git a/dev-requirements.txt b/dev-requirements.txt index 0d6af909a2..a9de992c14 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -8,6 +8,10 @@ # via # -c requirements.txt # pytest-flyte +absl-py==1.3.0 + # via + # tensorboard + # tensorflow arrow==1.2.3 # via # -c requirements.txt @@ -28,6 +32,8 @@ binaryornot==0.4.4 # via # -c requirements.txt # cookiecutter +cached-property==1.5.2 + # via docker-compose cachetools==5.2.0 # via google-auth certifi==2022.12.7 @@ -122,10 +128,14 @@ exceptiongroup==1.0.4 # via pytest filelock==3.8.2 # via virtualenv +flatbuffers==22.12.6 + # via tensorflow flyteidl==1.3.0 # via # -c requirements.txt # flytekit +gast==0.5.3 + # via tensorflow google-api-core[grpc]==2.11.0 # via # google-cloud-bigquery @@ -166,10 +176,11 @@ grpcio==1.51.1 # google-api-core # google-cloud-bigquery # grpcio-status + # tensorboard + # tensorflow grpcio-status==1.51.1 # via # -c requirements.txt - # -r dev-requirements.in # flytekit # google-api-core h5py==3.7.0 @@ -183,9 +194,15 @@ idna==3.4 importlib-metadata==5.1.0 # via # -c requirements.txt + # click # flytekit + # jsonschema # keyring # markdown + # pluggy + # pre-commit + # pytest + # virtualenv iniconfig==1.1.1 # via pytest ipython==7.34.0 @@ -276,19 +293,20 @@ nodeenv==1.7.0 numpy==1.21.6 # via # -c requirements.txt + # flytekit # h5py # keras-preprocessing # opt-einsum # pandas # pyarrow + # scikit-learn + # scipy # tensorboard # tensorflow oauthlib==3.2.2 # via requests-oauthlib opt-einsum==3.3.0 # via tensorflow - # scikit-learn - # scipy packaging==21.3 # via # -c requirements.txt @@ -466,6 +484,7 @@ six==1.16.0 # astunparse # dockerpty # google-auth + # google-pasta # jsonschema # keras-preprocessing # paramiko @@ -490,9 +509,9 @@ tensorflow==2.8.1 # via -r dev-requirements.in tensorflow-estimator==2.8.0 # via tensorflow -tensorflow-io-gcs-filesystem==0.27.0 +tensorflow-io-gcs-filesystem==0.28.0 # via tensorflow -termcolor==2.0.1 +termcolor==2.1.1 # via tensorflow text-unidecode==1.3 # via @@ -527,8 +546,11 @@ types-toml==0.10.8.1 typing-extensions==4.4.0 # via # -c requirements.txt + # arrow # flytekit + # importlib-metadata # mypy + # responses # tensorflow # torch # typing-inspect @@ -552,6 +574,8 @@ websocket-client==0.59.0 # -c requirements.txt # docker # docker-compose +werkzeug==2.2.2 + # via tensorboard wheel==0.38.4 # via # -c requirements.txt @@ -563,6 +587,7 @@ wrapt==1.14.1 # -c requirements.txt # deprecated # flytekit + # tensorflow zipp==3.11.0 # via # -c requirements.txt diff --git a/requirements.in b/requirements.in index 0c3ba0378d..09828957fe 100644 --- a/requirements.in +++ b/requirements.in @@ -13,3 +13,5 @@ pandas<1.4.0 numpy<1.22.0 # This is required by docker-compose and otherwise clashes with docker-py websocket-client<1.0.0 +# TODO: Remove after buf migration is done and packages updated +packaging<22.0 diff --git a/requirements.txt b/requirements.txt index bb994fb958..caff0db497 100644 --- a/requirements.txt +++ b/requirements.txt @@ -113,6 +113,7 @@ numpy==1.21.6 # pyarrow packaging==21.3 # via + # -r requirements.in # docker # marshmallow pandas==1.3.5 From a025d07b33668a49f1b5158b1ef9636a2bb0d0ed Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Wed, 7 Dec 2022 23:46:52 -0800 Subject: [PATCH 6/8] Set env var in doc generation Signed-off-by: Eduardo Apolinario --- .github/workflows/pythonbuild.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pythonbuild.yml b/.github/workflows/pythonbuild.yml index fc8a554cdd..8110c29141 100644 --- a/.github/workflows/pythonbuild.yml +++ b/.github/workflows/pythonbuild.yml @@ -177,4 +177,6 @@ jobs: python -m pip install --upgrade pip==21.2.4 setuptools wheel pip install -r doc-requirements.txt - name: Build the documentation - run: make -C docs html + run: | + # TODO: Remove after buf migration is done and packages updated + PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python make -C docs html From 6885010381b71eef8b0a0cce0e5403ffad51f13c Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Thu, 8 Dec 2022 00:30:22 -0800 Subject: [PATCH 7/8] Add unit_test_codecov make target Signed-off-by: Eduardo Apolinario --- .github/workflows/pythonbuild.yml | 2 +- Makefile | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pythonbuild.yml b/.github/workflows/pythonbuild.yml index 8110c29141..0f47137169 100644 --- a/.github/workflows/pythonbuild.yml +++ b/.github/workflows/pythonbuild.yml @@ -49,7 +49,7 @@ jobs: pip freeze - name: Test with coverage run: | - pytest tests/flytekit/unit -m "not sandbox_test" --cov=./ --cov-report=xml + make unit_test_codecov - name: Codecov uses: codecov/codecov-action@v3.1.0 with: diff --git a/Makefile b/Makefile index 2f71c97ebe..32947c21ca 100644 --- a/Makefile +++ b/Makefile @@ -49,12 +49,17 @@ spellcheck: ## Runs a spellchecker over all code and documentation .PHONY: test test: lint unit_test +.PHONY: unit_test_codecov +unit_test_codecov: + $(MAKE) CODECOV_OPTS="--cov=./ --cov-report=xml" unit_test + .PHONY: unit_test unit_test: # Skip tensorflow tests and run them with the necessary env var set so that a working (albeit slower) # library is used to serialize/deserialize protobufs is used. - pytest -m "not sandbox_test" tests/flytekit/unit/ --ignore=tests/flytekit/unit/extras/tensorflow && \ - PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python pytest tests/flytekit/unit/extras/tensorflow + CODECOV_OPTS=${CODECOV_OPTS:-""} + pytest -m "not sandbox_test" tests/flytekit/unit/deck/ --ignore=tests/flytekit/unit/extras/tensorflow ${CODECOV_OPTS} && \ + PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python pytest tests/flytekit/unit/extras/tensorflow ${CODECOV_OPTS} requirements-spark2.txt: export CUSTOM_COMPILE_COMMAND := make requirements-spark2.txt requirements-spark2.txt: requirements-spark2.in install-piptools From 8c77c576b60e67f88b46ae5bc1aaf12e41e503e0 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Thu, 8 Dec 2022 00:52:51 -0800 Subject: [PATCH 8/8] Aggregate code coverage data and run all unit tests Signed-off-by: Eduardo Apolinario --- Makefile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 32947c21ca..8254904469 100644 --- a/Makefile +++ b/Makefile @@ -51,14 +51,15 @@ test: lint unit_test .PHONY: unit_test_codecov unit_test_codecov: - $(MAKE) CODECOV_OPTS="--cov=./ --cov-report=xml" unit_test + # Ensure coverage file + rm coverage.xml || true + $(MAKE) CODECOV_OPTS="--cov=./ --cov-report=xml --cov-append" unit_test .PHONY: unit_test unit_test: # Skip tensorflow tests and run them with the necessary env var set so that a working (albeit slower) # library is used to serialize/deserialize protobufs is used. - CODECOV_OPTS=${CODECOV_OPTS:-""} - pytest -m "not sandbox_test" tests/flytekit/unit/deck/ --ignore=tests/flytekit/unit/extras/tensorflow ${CODECOV_OPTS} && \ + pytest -m "not sandbox_test" tests/flytekit/unit/ --ignore=tests/flytekit/unit/extras/tensorflow ${CODECOV_OPTS} && \ PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python pytest tests/flytekit/unit/extras/tensorflow ${CODECOV_OPTS} requirements-spark2.txt: export CUSTOM_COMPILE_COMMAND := make requirements-spark2.txt