From d6a8c5df5e8d6d6b7b832ff9aa12ea0464bbaed1 Mon Sep 17 00:00:00 2001 From: Sean Lin Date: Fri, 10 Sep 2021 13:57:04 -0700 Subject: [PATCH] #minor Revert "Adopt flyteidl's ordered variable map change" #608 (#647) * Revert "Adopt flyteidl's ordered variable map change (#608)" This reverts commit 204b13f4 * Upgrade flyteidl to 0.21.0 Signed-off-by: Sean Lin --- .github/workflows/pythonbuild.yml | 3 +- dev-requirements.txt | 12 ++--- doc-requirements.txt | 16 +++--- flytekit/models/interface.py | 30 ++++------- .../sqlalchemy/requirements.in | 2 +- .../sqlalchemy/requirements.txt | 10 ++-- requirements-spark2.txt | 12 ++--- requirements.txt | 12 ++--- setup.py | 2 +- .../mock_flyte_repo/workflows/requirements.in | 2 +- .../workflows/requirements.txt | 54 +++++++++---------- .../unit/common_tests/test_launch_plan.py | 4 +- .../unit/common_tests/test_workflow.py | 8 +-- 13 files changed, 71 insertions(+), 96 deletions(-) diff --git a/.github/workflows/pythonbuild.yml b/.github/workflows/pythonbuild.yml index f1c8551ffc..68d8f095d7 100644 --- a/.github/workflows/pythonbuild.yml +++ b/.github/workflows/pythonbuild.yml @@ -34,7 +34,6 @@ jobs: run: | python -m pip install pip==21.2.4 setuptools==57.4.0 wheel==0.37.0 make setup${{ matrix.spark-version-suffix }} - make -C plugins install-all-dev pip freeze - name: Lint run: | @@ -46,7 +45,7 @@ jobs: boilerplate - name: Test with coverage run: | - coverage run -m pytest tests/flytekit/unit tests/scripts plugins/tests + coverage run -m pytest tests/flytekit/unit tests/scripts - name: Integration Tests with coverage # https://github.com/actions/runner/issues/241#issuecomment-577360161 shell: 'script -q -e -c "bash {0}"' diff --git a/dev-requirements.txt b/dev-requirements.txt index 5586dfff9c..67a2eac899 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -61,7 +61,7 @@ decorator==5.0.9 # via # -c requirements.txt # retry -deprecated==1.2.12 +deprecated==1.2.13 # via # -c requirements.txt # flytekit @@ -77,7 +77,7 @@ distlib==0.3.2 # via virtualenv distro==1.6.0 # via docker-compose -docker[ssh]==5.0.1 +docker[ssh]==5.0.2 # via docker-compose docker-compose==1.29.2 # via @@ -97,15 +97,15 @@ docstring-parser==0.10 # flytekit filelock==3.0.12 # via virtualenv -flyteidl==0.20.1 +flyteidl==0.21.0 # via # -c requirements.txt # flytekit -grpcio==1.39.0 +grpcio==1.40.0 # via # -c requirements.txt # flytekit -identify==2.2.13 +identify==2.2.14 # via pre-commit idna==3.2 # via @@ -189,7 +189,7 @@ platformdirs==2.3.0 # virtualenv pluggy==1.0.0 # via pytest -pre-commit==2.14.1 +pre-commit==2.15.0 # via -r dev-requirements.in protobuf==3.17.3 # via diff --git a/doc-requirements.txt b/doc-requirements.txt index 9cd6c53e4a..e1d9448ede 100644 --- a/doc-requirements.txt +++ b/doc-requirements.txt @@ -26,7 +26,7 @@ backcall==0.2.0 # via ipython bcrypt==3.2.0 # via paramiko -beautifulsoup4==4.9.3 +beautifulsoup4==4.10.0 # via # furo # sphinx-code-include @@ -35,9 +35,9 @@ black==21.8b0 # via papermill bleach==4.1.0 # via nbconvert -boto3==1.18.32 +boto3==1.18.39 # via sagemaker-training -botocore==1.21.32 +botocore==1.21.39 # via # boto3 # s3transfer @@ -72,7 +72,7 @@ decorator==5.0.9 # retry defusedxml==0.7.1 # via nbconvert -deprecated==1.2.12 +deprecated==1.2.13 # via flytekit dirhash==0.2.1 # via flytekit @@ -89,7 +89,7 @@ entrypoints==0.3 # jupyter-client # nbconvert # papermill -flyteidl==0.20.1 +flyteidl==0.21.0 # via flytekit git+git://github.com/flyteorg/furo@main # via -r doc-requirements.in @@ -97,7 +97,7 @@ gevent==21.8.0 # via sagemaker-training greenlet==1.1.1 # via gevent -grpcio==1.39.0 +grpcio==1.40.0 # via # -r doc-requirements.in # flytekit @@ -160,7 +160,7 @@ marshmallow-enum==1.5.1 # via dataclasses-json marshmallow-jsonschema==0.12.0 # via flytekit -matplotlib-inline==0.1.2 +matplotlib-inline==0.1.3 # via ipython mistune==0.8.4 # via nbconvert @@ -397,7 +397,7 @@ typing-extensions==3.10.0.2 # typing-inspect typing-inspect==0.7.1 # via dataclasses-json -unidecode==1.2.0 +unidecode==1.3.1 # via # python-slugify # sphinx-autoapi diff --git a/flytekit/models/interface.py b/flytekit/models/interface.py index 9de8dd2ad5..5364d39c3e 100644 --- a/flytekit/models/interface.py +++ b/flytekit/models/interface.py @@ -1,6 +1,6 @@ import typing -from collections import OrderedDict +import six as _six from flyteidl.core import interface_pb2 as _interface_pb2 from flytekit.models import common as _common @@ -73,9 +73,7 @@ def to_flyte_idl(self): """ :rtype: dict[Text, Variable] """ - return _interface_pb2.VariableMap( - variables=[_interface_pb2.VariableMapEntry(name=k, var=v.to_flyte_idl()) for k, v in self.variables.items()] - ) + return _interface_pb2.VariableMap(variables={k: v.to_flyte_idl() for k, v in _six.iteritems(self.variables)}) @classmethod def from_flyte_idl(cls, pb2_object): @@ -83,7 +81,7 @@ def from_flyte_idl(cls, pb2_object): :param dict[Text, Variable] pb2_object: :rtype: VariableMap """ - return cls(OrderedDict((v.name, Variable.from_flyte_idl(v.var)) for v in pb2_object.variables)) + return cls({k: Variable.from_flyte_idl(v) for k, v in _six.iteritems(pb2_object.variables)}) class TypedInterface(_common.FlyteIdlEntity): @@ -108,15 +106,9 @@ def outputs(self) -> typing.Dict[str, Variable]: def to_flyte_idl(self) -> _interface_pb2.TypedInterface: return _interface_pb2.TypedInterface( - inputs=_interface_pb2.VariableMap( - variables=[ - _interface_pb2.VariableMapEntry(name=k, var=v.to_flyte_idl()) for k, v in self.inputs.items() - ] - ), + inputs=_interface_pb2.VariableMap(variables={k: v.to_flyte_idl() for k, v in _six.iteritems(self.inputs)}), outputs=_interface_pb2.VariableMap( - variables=[ - _interface_pb2.VariableMapEntry(name=k, var=v.to_flyte_idl()) for k, v in self.outputs.items() - ] + variables={k: v.to_flyte_idl() for k, v in _six.iteritems(self.outputs)} ), ) @@ -126,8 +118,8 @@ def from_flyte_idl(cls, proto: _interface_pb2.TypedInterface) -> "TypedInterface :param proto: """ return cls( - inputs=OrderedDict((v.name, Variable.from_flyte_idl(v.var)) for v in proto.inputs.variables), - outputs=OrderedDict((v.name, Variable.from_flyte_idl(v.var)) for v in proto.outputs.variables), + inputs={k: Variable.from_flyte_idl(v) for k, v in _six.iteritems(proto.inputs.variables)}, + outputs={k: Variable.from_flyte_idl(v) for k, v in _six.iteritems(proto.outputs.variables)}, ) @@ -219,9 +211,7 @@ def to_flyte_idl(self): :rtype: flyteidl.core.interface_pb2.ParameterMap """ return _interface_pb2.ParameterMap( - parameters=[ - _interface_pb2.ParameterMapEntry(name=k, parameter=v.to_flyte_idl()) for k, v in self.parameters.items() - ] + parameters={k: v.to_flyte_idl() for k, v in _six.iteritems(self.parameters)}, ) @classmethod @@ -230,6 +220,4 @@ def from_flyte_idl(cls, pb2_object): :param flyteidl.core.interface_pb2.ParameterMap pb2_object: :rtype: ParameterMap """ - return cls( - parameters=OrderedDict((v.name, Parameter.from_flyte_idl(v.parameter)) for v in pb2_object.parameters) - ) + return cls(parameters={k: Parameter.from_flyte_idl(v) for k, v in _six.iteritems(pb2_object.parameters)}) diff --git a/plugins/flytekit-sqlalchemy/flytekitplugins/sqlalchemy/requirements.in b/plugins/flytekit-sqlalchemy/flytekitplugins/sqlalchemy/requirements.in index 594c9deb03..e06c9fb7d8 100644 --- a/plugins/flytekit-sqlalchemy/flytekitplugins/sqlalchemy/requirements.in +++ b/plugins/flytekit-sqlalchemy/flytekitplugins/sqlalchemy/requirements.in @@ -1,2 +1,2 @@ -flytekit>=0.22.0b6 +flytekit>=0.20.1 sqlalchemy diff --git a/plugins/flytekit-sqlalchemy/flytekitplugins/sqlalchemy/requirements.txt b/plugins/flytekit-sqlalchemy/flytekitplugins/sqlalchemy/requirements.txt index cc9d7a9d9e..524cf6a0db 100644 --- a/plugins/flytekit-sqlalchemy/flytekitplugins/sqlalchemy/requirements.txt +++ b/plugins/flytekit-sqlalchemy/flytekitplugins/sqlalchemy/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # pip-compile requirements.in @@ -26,15 +26,11 @@ deprecated==1.2.12 # via flytekit dirhash==0.2.1 # via flytekit -diskcache==5.2.1 - # via flytekit docker-image-py==0.1.10 # via flytekit -docstring-parser==0.10 - # via flytekit -flyteidl==0.20.1 +flyteidl==0.19.5 # via flytekit -flytekit==0.22.0b6 +flytekit==0.20.1 # via -r requirements.in greenlet==1.1.0 # via sqlalchemy diff --git a/requirements-spark2.txt b/requirements-spark2.txt index b1249159a2..4007626c78 100644 --- a/requirements-spark2.txt +++ b/requirements-spark2.txt @@ -25,9 +25,9 @@ black==21.8b0 # via papermill bleach==4.1.0 # via nbconvert -boto3==1.18.32 +boto3==1.18.39 # via sagemaker-training -botocore==1.21.32 +botocore==1.21.39 # via # boto3 # s3transfer @@ -58,7 +58,7 @@ decorator==5.0.9 # retry defusedxml==0.7.1 # via nbconvert -deprecated==1.2.12 +deprecated==1.2.13 # via flytekit dirhash==0.2.1 # via flytekit @@ -73,13 +73,13 @@ entrypoints==0.3 # jupyter-client # nbconvert # papermill -flyteidl==0.20.1 +flyteidl==0.21.0 # via flytekit gevent==21.8.0 # via sagemaker-training greenlet==1.1.1 # via gevent -grpcio==1.39.0 +grpcio==1.40.0 # via flytekit hmsclient==0.1.1 # via flytekit @@ -131,7 +131,7 @@ marshmallow-enum==1.5.1 # via dataclasses-json marshmallow-jsonschema==0.12.0 # via flytekit -matplotlib-inline==0.1.2 +matplotlib-inline==0.1.3 # via ipython mistune==0.8.4 # via nbconvert diff --git a/requirements.txt b/requirements.txt index d7347fa5fc..d6d855e162 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,9 +25,9 @@ black==21.8b0 # via papermill bleach==4.1.0 # via nbconvert -boto3==1.18.32 +boto3==1.18.39 # via sagemaker-training -botocore==1.21.32 +botocore==1.21.39 # via # boto3 # s3transfer @@ -58,7 +58,7 @@ decorator==5.0.9 # retry defusedxml==0.7.1 # via nbconvert -deprecated==1.2.12 +deprecated==1.2.13 # via flytekit dirhash==0.2.1 # via flytekit @@ -73,13 +73,13 @@ entrypoints==0.3 # jupyter-client # nbconvert # papermill -flyteidl==0.20.1 +flyteidl==0.21.0 # via flytekit gevent==21.8.0 # via sagemaker-training greenlet==1.1.1 # via gevent -grpcio==1.39.0 +grpcio==1.40.0 # via flytekit hmsclient==0.1.1 # via flytekit @@ -131,7 +131,7 @@ marshmallow-enum==1.5.1 # via dataclasses-json marshmallow-jsonschema==0.12.0 # via flytekit -matplotlib-inline==0.1.2 +matplotlib-inline==0.1.3 # via ipython mistune==0.8.4 # via nbconvert diff --git a/setup.py b/setup.py index 410ad7cc76..6fff1a7d8c 100644 --- a/setup.py +++ b/setup.py @@ -64,7 +64,7 @@ ] }, install_requires=[ - "flyteidl>=0.20.1,<0.21.0", + "flyteidl>=0.21.0,<0.22.0", "wheel>=0.30.0,<1.0.0", "pandas>=1.0.0,<2.0.0", "pyarrow>=2.0.0,<4.0.0", diff --git a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in index 5adb2ab9c5..f7d015b843 100644 --- a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in +++ b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in @@ -1,4 +1,4 @@ -flytekit>=0.22.0b6 +flytekit>=0.17.0b0 wheel matplotlib opencv-python diff --git a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt index c18f52c51a..af545501da 100644 --- a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt +++ b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt @@ -8,7 +8,7 @@ attrs==21.2.0 # via scantree certifi==2021.5.30 # via requests -charset-normalizer==2.0.4 +chardet==4.0.0 # via requests click==7.1.2 # via flytekit @@ -16,35 +16,31 @@ croniter==1.0.15 # via flytekit cycler==0.10.0 # via matplotlib -dataclasses-json==0.5.5 +dataclasses-json==0.5.4 # via flytekit decorator==5.0.9 # via retry -deprecated==1.2.13 +deprecated==1.2.12 # via flytekit dirhash==0.2.1 # via flytekit -diskcache==5.2.1 +docker-image-py==0.1.10 # via flytekit -docker-image-py==0.1.12 +flyteidl==0.19.11 # via flytekit -docstring-parser==0.10 - # via flytekit -flyteidl==0.20.2 - # via flytekit -flytekit==0.22.1 +flytekit==0.20.1 # via -r requirements.in -grpcio==1.39.0 +grpcio==1.38.1 # via flytekit -idna==3.2 +idna==2.10 # via requests -importlib-metadata==4.8.1 +importlib-metadata==4.6.1 # via keyring -keyring==23.1.0 +keyring==23.0.1 # via flytekit -kiwisolver==1.3.2 +kiwisolver==1.3.1 # via matplotlib -marshmallow==3.13.0 +marshmallow==3.12.2 # via # dataclasses-json # marshmallow-enum @@ -53,25 +49,25 @@ marshmallow-enum==1.5.1 # via dataclasses-json marshmallow-jsonschema==0.12.0 # via flytekit -matplotlib==3.4.3 +matplotlib==3.4.2 # via -r requirements.in mypy-extensions==0.4.3 # via typing-inspect natsort==7.1.1 # via flytekit -numpy==1.21.2 +numpy==1.21.0 # via # matplotlib # opencv-python # pandas # pyarrow -opencv-python==4.5.3.56 +opencv-python==4.5.2.54 # via -r requirements.in -pandas==1.3.2 +pandas==1.3.0 # via flytekit -pathspec==0.9.0 +pathspec==0.8.1 # via scantree -pillow==8.3.2 +pillow==8.3.1 # via matplotlib protobuf==3.17.3 # via @@ -89,7 +85,7 @@ python-dateutil==2.8.1 # flytekit # matplotlib # pandas -python-json-logger==2.0.2 +python-json-logger==2.0.1 # via flytekit pytimeparse==1.1.8 # via flytekit @@ -97,13 +93,13 @@ pytz==2018.4 # via # flytekit # pandas -regex==2021.8.28 +regex==2021.7.6 # via docker-image-py -requests==2.26.0 +requests==2.25.1 # via # flytekit # responses -responses==0.13.4 +responses==0.13.3 # via flytekit retry==0.9.2 # via flytekit @@ -122,7 +118,9 @@ sortedcontainers==2.4.0 # via flytekit statsd==3.3.0 # via flytekit -typing-extensions==3.10.0.2 +stringcase==1.2.0 + # via dataclasses-json +typing-extensions==3.10.0.0 # via typing-inspect typing-inspect==0.7.1 # via dataclasses-json @@ -131,7 +129,7 @@ urllib3==1.26.6 # flytekit # requests # responses -wheel==0.37.0 +wheel==0.36.2 # via # -r requirements.in # flytekit diff --git a/tests/flytekit/unit/common_tests/test_launch_plan.py b/tests/flytekit/unit/common_tests/test_launch_plan.py index 8a90d4d652..92943d37f1 100644 --- a/tests/flytekit/unit/common_tests/test_launch_plan.py +++ b/tests/flytekit/unit/common_tests/test_launch_plan.py @@ -340,9 +340,7 @@ def test_serialize(): == _identifier.Identifier(_identifier.ResourceType.WORKFLOW, "p", "d", "n", "v").to_flyte_idl() ) assert s.spec.auth_role.assumable_iam_role == "iam_role" - assert len(s.spec.default_inputs.parameters) == 1 - assert s.spec.default_inputs.parameters[0].name == "default_input" - assert s.spec.default_inputs.parameters[0].parameter.default.scalar.primitive.integer == 5 + assert s.spec.default_inputs.parameters["default_input"].default.scalar.primitive.integer == 5 def test_promote_from_model(): diff --git a/tests/flytekit/unit/common_tests/test_workflow.py b/tests/flytekit/unit/common_tests/test_workflow.py index 788d960005..13a9e65d8e 100644 --- a/tests/flytekit/unit/common_tests/test_workflow.py +++ b/tests/flytekit/unit/common_tests/test_workflow.py @@ -366,12 +366,8 @@ def my_list_task(wf_params, a, b): serialized = w.serialize() assert isinstance(serialized, _workflow_pb2.WorkflowSpec) assert len(serialized.template.nodes) == 6 - assert len(serialized.template.interface.inputs.variables) == 2 - assert len(serialized.template.interface.outputs.variables) == 2 - assert serialized.template.interface.inputs.variables[0].name == "required" - assert serialized.template.interface.inputs.variables[1].name == "not_required" - assert serialized.template.interface.outputs.variables[0].name == "nested_out" - assert serialized.template.interface.outputs.variables[1].name == "scalar_out" + assert len(serialized.template.interface.inputs.variables.keys()) == 2 + assert len(serialized.template.interface.outputs.variables.keys()) == 2 def test_workflow_disable_default_launch_plan():