From 204b13f4e8fab08fdb7e9f8789c86bc1a5369447 Mon Sep 17 00:00:00 2001 From: Sean Lin Date: Tue, 31 Aug 2021 16:49:08 -0700 Subject: [PATCH] Adopt flyteidl's ordered variable map change (#608) Signed-off-by: Sean Lin Signed-off-by: Yee Hing Tong Signed-off-by: Eduardo Apolinario Co-authored-by: Yee Hing Tong Co-authored-by: Eduardo Apolinario --- dev-requirements.txt | 6 +-- doc-requirements.txt | 4 +- flytekit/models/interface.py | 30 +++++++---- .../sqlalchemy/requirements.in | 2 +- .../sqlalchemy/requirements.txt | 10 ++-- requirements-spark2.txt | 4 +- requirements.txt | 4 +- setup.py | 2 +- .../mock_flyte_repo/workflows/requirements.in | 2 +- .../workflows/requirements.txt | 50 ++++++++++--------- .../unit/common_tests/test_launch_plan.py | 4 +- .../unit/common_tests/test_workflow.py | 8 ++- 12 files changed, 75 insertions(+), 51 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 3ce16c3ede..5586dfff9c 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -77,7 +77,7 @@ distlib==0.3.2 # via virtualenv distro==1.6.0 # via docker-compose -docker[ssh]==5.0.0 +docker[ssh]==5.0.1 # via docker-compose docker-compose==1.29.2 # via @@ -97,7 +97,7 @@ docstring-parser==0.10 # flytekit filelock==3.0.12 # via virtualenv -flyteidl==0.19.25 +flyteidl==0.20.1 # via # -c requirements.txt # flytekit @@ -183,7 +183,7 @@ pathspec==0.9.0 # via # -c requirements.txt # scantree -platformdirs==2.2.0 +platformdirs==2.3.0 # via # -c requirements.txt # virtualenv diff --git a/doc-requirements.txt b/doc-requirements.txt index cc4647a9d9..9cd6c53e4a 100644 --- a/doc-requirements.txt +++ b/doc-requirements.txt @@ -89,7 +89,7 @@ entrypoints==0.3 # jupyter-client # nbconvert # papermill -flyteidl==0.19.25 +flyteidl==0.20.1 # via flytekit git+git://github.com/flyteorg/furo@main # via -r doc-requirements.in @@ -214,7 +214,7 @@ pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -platformdirs==2.2.0 +platformdirs==2.3.0 # via black prompt-toolkit==3.0.20 # via ipython diff --git a/flytekit/models/interface.py b/flytekit/models/interface.py index 5364d39c3e..9de8dd2ad5 100644 --- a/flytekit/models/interface.py +++ b/flytekit/models/interface.py @@ -1,6 +1,6 @@ import typing +from collections import OrderedDict -import six as _six from flyteidl.core import interface_pb2 as _interface_pb2 from flytekit.models import common as _common @@ -73,7 +73,9 @@ def to_flyte_idl(self): """ :rtype: dict[Text, Variable] """ - return _interface_pb2.VariableMap(variables={k: v.to_flyte_idl() for k, v in _six.iteritems(self.variables)}) + return _interface_pb2.VariableMap( + variables=[_interface_pb2.VariableMapEntry(name=k, var=v.to_flyte_idl()) for k, v in self.variables.items()] + ) @classmethod def from_flyte_idl(cls, pb2_object): @@ -81,7 +83,7 @@ def from_flyte_idl(cls, pb2_object): :param dict[Text, Variable] pb2_object: :rtype: VariableMap """ - return cls({k: Variable.from_flyte_idl(v) for k, v in _six.iteritems(pb2_object.variables)}) + return cls(OrderedDict((v.name, Variable.from_flyte_idl(v.var)) for v in pb2_object.variables)) class TypedInterface(_common.FlyteIdlEntity): @@ -106,9 +108,15 @@ def outputs(self) -> typing.Dict[str, Variable]: def to_flyte_idl(self) -> _interface_pb2.TypedInterface: return _interface_pb2.TypedInterface( - inputs=_interface_pb2.VariableMap(variables={k: v.to_flyte_idl() for k, v in _six.iteritems(self.inputs)}), + inputs=_interface_pb2.VariableMap( + variables=[ + _interface_pb2.VariableMapEntry(name=k, var=v.to_flyte_idl()) for k, v in self.inputs.items() + ] + ), outputs=_interface_pb2.VariableMap( - variables={k: v.to_flyte_idl() for k, v in _six.iteritems(self.outputs)} + variables=[ + _interface_pb2.VariableMapEntry(name=k, var=v.to_flyte_idl()) for k, v in self.outputs.items() + ] ), ) @@ -118,8 +126,8 @@ def from_flyte_idl(cls, proto: _interface_pb2.TypedInterface) -> "TypedInterface :param proto: """ return cls( - inputs={k: Variable.from_flyte_idl(v) for k, v in _six.iteritems(proto.inputs.variables)}, - outputs={k: Variable.from_flyte_idl(v) for k, v in _six.iteritems(proto.outputs.variables)}, + inputs=OrderedDict((v.name, Variable.from_flyte_idl(v.var)) for v in proto.inputs.variables), + outputs=OrderedDict((v.name, Variable.from_flyte_idl(v.var)) for v in proto.outputs.variables), ) @@ -211,7 +219,9 @@ def to_flyte_idl(self): :rtype: flyteidl.core.interface_pb2.ParameterMap """ return _interface_pb2.ParameterMap( - parameters={k: v.to_flyte_idl() for k, v in _six.iteritems(self.parameters)}, + parameters=[ + _interface_pb2.ParameterMapEntry(name=k, parameter=v.to_flyte_idl()) for k, v in self.parameters.items() + ] ) @classmethod @@ -220,4 +230,6 @@ def from_flyte_idl(cls, pb2_object): :param flyteidl.core.interface_pb2.ParameterMap pb2_object: :rtype: ParameterMap """ - return cls(parameters={k: Parameter.from_flyte_idl(v) for k, v in _six.iteritems(pb2_object.parameters)}) + return cls( + parameters=OrderedDict((v.name, Parameter.from_flyte_idl(v.parameter)) for v in pb2_object.parameters) + ) diff --git a/plugins/flytekit-sqlalchemy/flytekitplugins/sqlalchemy/requirements.in b/plugins/flytekit-sqlalchemy/flytekitplugins/sqlalchemy/requirements.in index e06c9fb7d8..594c9deb03 100644 --- a/plugins/flytekit-sqlalchemy/flytekitplugins/sqlalchemy/requirements.in +++ b/plugins/flytekit-sqlalchemy/flytekitplugins/sqlalchemy/requirements.in @@ -1,2 +1,2 @@ -flytekit>=0.20.1 +flytekit>=0.22.0b6 sqlalchemy diff --git a/plugins/flytekit-sqlalchemy/flytekitplugins/sqlalchemy/requirements.txt b/plugins/flytekit-sqlalchemy/flytekitplugins/sqlalchemy/requirements.txt index 524cf6a0db..cc9d7a9d9e 100644 --- a/plugins/flytekit-sqlalchemy/flytekitplugins/sqlalchemy/requirements.txt +++ b/plugins/flytekit-sqlalchemy/flytekitplugins/sqlalchemy/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.8 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # pip-compile requirements.in @@ -26,11 +26,15 @@ deprecated==1.2.12 # via flytekit dirhash==0.2.1 # via flytekit +diskcache==5.2.1 + # via flytekit docker-image-py==0.1.10 # via flytekit -flyteidl==0.19.5 +docstring-parser==0.10 + # via flytekit +flyteidl==0.20.1 # via flytekit -flytekit==0.20.1 +flytekit==0.22.0b6 # via -r requirements.in greenlet==1.1.0 # via sqlalchemy diff --git a/requirements-spark2.txt b/requirements-spark2.txt index 15c294d418..b1249159a2 100644 --- a/requirements-spark2.txt +++ b/requirements-spark2.txt @@ -73,7 +73,7 @@ entrypoints==0.3 # jupyter-client # nbconvert # papermill -flyteidl==0.19.25 +flyteidl==0.20.1 # via flytekit gevent==21.8.0 # via sagemaker-training @@ -183,7 +183,7 @@ pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -platformdirs==2.2.0 +platformdirs==2.3.0 # via black prompt-toolkit==3.0.20 # via ipython diff --git a/requirements.txt b/requirements.txt index febd370af9..d7347fa5fc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -73,7 +73,7 @@ entrypoints==0.3 # jupyter-client # nbconvert # papermill -flyteidl==0.19.25 +flyteidl==0.20.1 # via flytekit gevent==21.8.0 # via sagemaker-training @@ -183,7 +183,7 @@ pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -platformdirs==2.2.0 +platformdirs==2.3.0 # via black prompt-toolkit==3.0.20 # via ipython diff --git a/setup.py b/setup.py index cc0b9604f6..410ad7cc76 100644 --- a/setup.py +++ b/setup.py @@ -64,7 +64,7 @@ ] }, install_requires=[ - "flyteidl>=0.19.18,<0.20.0", + "flyteidl>=0.20.1,<0.21.0", "wheel>=0.30.0,<1.0.0", "pandas>=1.0.0,<2.0.0", "pyarrow>=2.0.0,<4.0.0", diff --git a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in index f7d015b843..5adb2ab9c5 100644 --- a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in +++ b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in @@ -1,4 +1,4 @@ -flytekit>=0.17.0b0 +flytekit>=0.22.0b6 wheel matplotlib opencv-python diff --git a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt index af545501da..e6a22517e5 100644 --- a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt +++ b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt @@ -8,7 +8,7 @@ attrs==21.2.0 # via scantree certifi==2021.5.30 # via requests -chardet==4.0.0 +charset-normalizer==2.0.4 # via requests click==7.1.2 # via flytekit @@ -16,7 +16,7 @@ croniter==1.0.15 # via flytekit cycler==0.10.0 # via matplotlib -dataclasses-json==0.5.4 +dataclasses-json==0.5.5 # via flytekit decorator==5.0.9 # via retry @@ -24,23 +24,27 @@ deprecated==1.2.12 # via flytekit dirhash==0.2.1 # via flytekit -docker-image-py==0.1.10 +diskcache==5.2.1 # via flytekit -flyteidl==0.19.11 +docker-image-py==0.1.12 # via flytekit -flytekit==0.20.1 +docstring-parser==0.10 + # via flytekit +flyteidl==0.20.1 + # via flytekit +flytekit==0.22.0b6 # via -r requirements.in -grpcio==1.38.1 +grpcio==1.39.0 # via flytekit -idna==2.10 +idna==3.2 # via requests -importlib-metadata==4.6.1 +importlib-metadata==4.8.1 # via keyring -keyring==23.0.1 +keyring==23.1.0 # via flytekit -kiwisolver==1.3.1 +kiwisolver==1.3.2 # via matplotlib -marshmallow==3.12.2 +marshmallow==3.13.0 # via # dataclasses-json # marshmallow-enum @@ -49,23 +53,23 @@ marshmallow-enum==1.5.1 # via dataclasses-json marshmallow-jsonschema==0.12.0 # via flytekit -matplotlib==3.4.2 +matplotlib==3.4.3 # via -r requirements.in mypy-extensions==0.4.3 # via typing-inspect natsort==7.1.1 # via flytekit -numpy==1.21.0 +numpy==1.21.2 # via # matplotlib # opencv-python # pandas # pyarrow -opencv-python==4.5.2.54 +opencv-python==4.5.3.56 # via -r requirements.in -pandas==1.3.0 +pandas==1.3.2 # via flytekit -pathspec==0.8.1 +pathspec==0.9.0 # via scantree pillow==8.3.1 # via matplotlib @@ -85,7 +89,7 @@ python-dateutil==2.8.1 # flytekit # matplotlib # pandas -python-json-logger==2.0.1 +python-json-logger==2.0.2 # via flytekit pytimeparse==1.1.8 # via flytekit @@ -93,13 +97,13 @@ pytz==2018.4 # via # flytekit # pandas -regex==2021.7.6 +regex==2021.8.28 # via docker-image-py -requests==2.25.1 +requests==2.26.0 # via # flytekit # responses -responses==0.13.3 +responses==0.13.4 # via flytekit retry==0.9.2 # via flytekit @@ -118,9 +122,7 @@ sortedcontainers==2.4.0 # via flytekit statsd==3.3.0 # via flytekit -stringcase==1.2.0 - # via dataclasses-json -typing-extensions==3.10.0.0 +typing-extensions==3.10.0.2 # via typing-inspect typing-inspect==0.7.1 # via dataclasses-json @@ -129,7 +131,7 @@ urllib3==1.26.6 # flytekit # requests # responses -wheel==0.36.2 +wheel==0.37.0 # via # -r requirements.in # flytekit diff --git a/tests/flytekit/unit/common_tests/test_launch_plan.py b/tests/flytekit/unit/common_tests/test_launch_plan.py index 92943d37f1..8a90d4d652 100644 --- a/tests/flytekit/unit/common_tests/test_launch_plan.py +++ b/tests/flytekit/unit/common_tests/test_launch_plan.py @@ -340,7 +340,9 @@ def test_serialize(): == _identifier.Identifier(_identifier.ResourceType.WORKFLOW, "p", "d", "n", "v").to_flyte_idl() ) assert s.spec.auth_role.assumable_iam_role == "iam_role" - assert s.spec.default_inputs.parameters["default_input"].default.scalar.primitive.integer == 5 + assert len(s.spec.default_inputs.parameters) == 1 + assert s.spec.default_inputs.parameters[0].name == "default_input" + assert s.spec.default_inputs.parameters[0].parameter.default.scalar.primitive.integer == 5 def test_promote_from_model(): diff --git a/tests/flytekit/unit/common_tests/test_workflow.py b/tests/flytekit/unit/common_tests/test_workflow.py index 13a9e65d8e..788d960005 100644 --- a/tests/flytekit/unit/common_tests/test_workflow.py +++ b/tests/flytekit/unit/common_tests/test_workflow.py @@ -366,8 +366,12 @@ def my_list_task(wf_params, a, b): serialized = w.serialize() assert isinstance(serialized, _workflow_pb2.WorkflowSpec) assert len(serialized.template.nodes) == 6 - assert len(serialized.template.interface.inputs.variables.keys()) == 2 - assert len(serialized.template.interface.outputs.variables.keys()) == 2 + assert len(serialized.template.interface.inputs.variables) == 2 + assert len(serialized.template.interface.outputs.variables) == 2 + assert serialized.template.interface.inputs.variables[0].name == "required" + assert serialized.template.interface.inputs.variables[1].name == "not_required" + assert serialized.template.interface.outputs.variables[0].name == "nested_out" + assert serialized.template.interface.outputs.variables[1].name == "scalar_out" def test_workflow_disable_default_launch_plan():