Adopt flyteidl's ordered variable map change (#608)

Signed-off-by: Sean Lin <[email protected]> Signed-off-by: Yee Hing Tong <[email protected]> Signed-off-by: Eduardo Apolinario <[email protected]> Co-authored-by: Yee Hing Tong <[email protected]> Co-authored-by: Eduardo Apolinario <[email protected]>
flyteorg · Aug 31, 2021 · 204b13f · 204b13f
1 parent 44c74b9
commit 204b13f
Show file tree

Hide file tree

Showing 12 changed files with 75 additions and 51 deletions.
diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -77,7 +77,7 @@ distlib==0.3.2
     # via virtualenv
 distro==1.6.0
     # via docker-compose
-docker[ssh]==5.0.0
+docker[ssh]==5.0.1
     # via docker-compose
 docker-compose==1.29.2
     # via
@@ -97,7 +97,7 @@ docstring-parser==0.10
     #   flytekit
 filelock==3.0.12
     # via virtualenv
-flyteidl==0.19.25
+flyteidl==0.20.1
     # via
     #   -c requirements.txt
     #   flytekit
@@ -183,7 +183,7 @@ pathspec==0.9.0
     # via
     #   -c requirements.txt
     #   scantree
-platformdirs==2.2.0
+platformdirs==2.3.0
     # via
     #   -c requirements.txt
     #   virtualenv

diff --git a/doc-requirements.txt b/doc-requirements.txt
@@ -89,7 +89,7 @@ entrypoints==0.3
     #   jupyter-client
     #   nbconvert
     #   papermill
-flyteidl==0.19.25
+flyteidl==0.20.1
     # via flytekit
 git+git://github.com/flyteorg/furo@main
     # via -r doc-requirements.in
@@ -214,7 +214,7 @@ pexpect==4.8.0
     # via ipython
 pickleshare==0.7.5
     # via ipython
-platformdirs==2.2.0
+platformdirs==2.3.0
     # via black
 prompt-toolkit==3.0.20
     # via ipython

diff --git a/flytekit/models/interface.py b/flytekit/models/interface.py
@@ -1,6 +1,6 @@
 import typing
+from collections import OrderedDict
 
-import six as _six
 from flyteidl.core import interface_pb2 as _interface_pb2
 
 from flytekit.models import common as _common
@@ -73,15 +73,17 @@ def to_flyte_idl(self):
         """
         :rtype: dict[Text, Variable]
         """
-        return _interface_pb2.VariableMap(variables={k: v.to_flyte_idl() for k, v in _six.iteritems(self.variables)})
+        return _interface_pb2.VariableMap(
+            variables=[_interface_pb2.VariableMapEntry(name=k, var=v.to_flyte_idl()) for k, v in self.variables.items()]
+        )
 
     @classmethod
     def from_flyte_idl(cls, pb2_object):
         """
         :param dict[Text, Variable] pb2_object:
         :rtype: VariableMap
         """
-        return cls({k: Variable.from_flyte_idl(v) for k, v in _six.iteritems(pb2_object.variables)})
+        return cls(OrderedDict((v.name, Variable.from_flyte_idl(v.var)) for v in pb2_object.variables))
 
 
 class TypedInterface(_common.FlyteIdlEntity):
@@ -106,9 +108,15 @@ def outputs(self) -> typing.Dict[str, Variable]:
 
     def to_flyte_idl(self) -> _interface_pb2.TypedInterface:
         return _interface_pb2.TypedInterface(
-            inputs=_interface_pb2.VariableMap(variables={k: v.to_flyte_idl() for k, v in _six.iteritems(self.inputs)}),
+            inputs=_interface_pb2.VariableMap(
+                variables=[
+                    _interface_pb2.VariableMapEntry(name=k, var=v.to_flyte_idl()) for k, v in self.inputs.items()
+                ]
+            ),
             outputs=_interface_pb2.VariableMap(
-                variables={k: v.to_flyte_idl() for k, v in _six.iteritems(self.outputs)}
+                variables=[
+                    _interface_pb2.VariableMapEntry(name=k, var=v.to_flyte_idl()) for k, v in self.outputs.items()
+                ]
             ),
         )
 
@@ -118,8 +126,8 @@ def from_flyte_idl(cls, proto: _interface_pb2.TypedInterface) -> "TypedInterface
         :param proto:
         """
         return cls(
-            inputs={k: Variable.from_flyte_idl(v) for k, v in _six.iteritems(proto.inputs.variables)},
-            outputs={k: Variable.from_flyte_idl(v) for k, v in _six.iteritems(proto.outputs.variables)},
+            inputs=OrderedDict((v.name, Variable.from_flyte_idl(v.var)) for v in proto.inputs.variables),
+            outputs=OrderedDict((v.name, Variable.from_flyte_idl(v.var)) for v in proto.outputs.variables),
         )
 
 
@@ -211,7 +219,9 @@ def to_flyte_idl(self):
         :rtype: flyteidl.core.interface_pb2.ParameterMap
         """
         return _interface_pb2.ParameterMap(
-            parameters={k: v.to_flyte_idl() for k, v in _six.iteritems(self.parameters)},
+            parameters=[
+                _interface_pb2.ParameterMapEntry(name=k, parameter=v.to_flyte_idl()) for k, v in self.parameters.items()
+            ]
         )
 
     @classmethod
@@ -220,4 +230,6 @@ def from_flyte_idl(cls, pb2_object):
         :param flyteidl.core.interface_pb2.ParameterMap pb2_object:
         :rtype: ParameterMap
         """
-        return cls(parameters={k: Parameter.from_flyte_idl(v) for k, v in _six.iteritems(pb2_object.parameters)})
+        return cls(
+            parameters=OrderedDict((v.name, Parameter.from_flyte_idl(v.parameter)) for v in pb2_object.parameters)
+        )
diff --git a/plugins/flytekit-sqlalchemy/flytekitplugins/sqlalchemy/requirements.in b/plugins/flytekit-sqlalchemy/flytekitplugins/sqlalchemy/requirements.in
@@ -1,2 +1,2 @@
-flytekit>=0.20.1
+flytekit>=0.22.0b6
 sqlalchemy
diff --git a/plugins/flytekit-sqlalchemy/flytekitplugins/sqlalchemy/requirements.txt b/plugins/flytekit-sqlalchemy/flytekitplugins/sqlalchemy/requirements.txt
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with python 3.8
+# This file is autogenerated by pip-compile with python 3.9
 # To update, run:
 #
 #    pip-compile requirements.in
@@ -26,11 +26,15 @@ deprecated==1.2.12
     # via flytekit
 dirhash==0.2.1
     # via flytekit
+diskcache==5.2.1
+    # via flytekit
 docker-image-py==0.1.10
     # via flytekit
-flyteidl==0.19.5
+docstring-parser==0.10
+    # via flytekit
+flyteidl==0.20.1
     # via flytekit
-flytekit==0.20.1
+flytekit==0.22.0b6
     # via -r requirements.in
 greenlet==1.1.0
     # via sqlalchemy

diff --git a/requirements-spark2.txt b/requirements-spark2.txt
@@ -73,7 +73,7 @@ entrypoints==0.3
     #   jupyter-client
     #   nbconvert
     #   papermill
-flyteidl==0.19.25
+flyteidl==0.20.1
     # via flytekit
 gevent==21.8.0
     # via sagemaker-training
@@ -183,7 +183,7 @@ pexpect==4.8.0
     # via ipython
 pickleshare==0.7.5
     # via ipython
-platformdirs==2.2.0
+platformdirs==2.3.0
     # via black
 prompt-toolkit==3.0.20
     # via ipython

diff --git a/requirements.txt b/requirements.txt
@@ -73,7 +73,7 @@ entrypoints==0.3
     #   jupyter-client
     #   nbconvert
     #   papermill
-flyteidl==0.19.25
+flyteidl==0.20.1
     # via flytekit
 gevent==21.8.0
     # via sagemaker-training
@@ -183,7 +183,7 @@ pexpect==4.8.0
     # via ipython
 pickleshare==0.7.5
     # via ipython
-platformdirs==2.2.0
+platformdirs==2.3.0
     # via black
 prompt-toolkit==3.0.20
     # via ipython

diff --git a/setup.py b/setup.py
@@ -64,7 +64,7 @@
         ]
     },
     install_requires=[
-        "flyteidl>=0.19.18,<0.20.0",
+        "flyteidl>=0.20.1,<0.21.0",
         "wheel>=0.30.0,<1.0.0",
         "pandas>=1.0.0,<2.0.0",
         "pyarrow>=2.0.0,<4.0.0",

diff --git a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in
@@ -1,4 +1,4 @@
-flytekit>=0.17.0b0
+flytekit>=0.22.0b6
 wheel
 matplotlib
 opencv-python
diff --git a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt
@@ -8,39 +8,43 @@ attrs==21.2.0
     # via scantree
 certifi==2021.5.30
     # via requests
-chardet==4.0.0
+charset-normalizer==2.0.4
     # via requests
 click==7.1.2
     # via flytekit
 croniter==1.0.15
     # via flytekit
 cycler==0.10.0
     # via matplotlib
-dataclasses-json==0.5.4
+dataclasses-json==0.5.5
     # via flytekit
 decorator==5.0.9
     # via retry
 deprecated==1.2.12
     # via flytekit
 dirhash==0.2.1
     # via flytekit
-docker-image-py==0.1.10
+diskcache==5.2.1
     # via flytekit
-flyteidl==0.19.11
+docker-image-py==0.1.12
     # via flytekit
-flytekit==0.20.1
+docstring-parser==0.10
+    # via flytekit
+flyteidl==0.20.1
+    # via flytekit
+flytekit==0.22.0b6
     # via -r requirements.in
-grpcio==1.38.1
+grpcio==1.39.0
     # via flytekit
-idna==2.10
+idna==3.2
     # via requests
-importlib-metadata==4.6.1
+importlib-metadata==4.8.1
     # via keyring
-keyring==23.0.1
+keyring==23.1.0
     # via flytekit
-kiwisolver==1.3.1
+kiwisolver==1.3.2
     # via matplotlib
-marshmallow==3.12.2
+marshmallow==3.13.0
     # via
     #   dataclasses-json
     #   marshmallow-enum
@@ -49,23 +53,23 @@ marshmallow-enum==1.5.1
     # via dataclasses-json
 marshmallow-jsonschema==0.12.0
     # via flytekit
-matplotlib==3.4.2
+matplotlib==3.4.3
     # via -r requirements.in
 mypy-extensions==0.4.3
     # via typing-inspect
 natsort==7.1.1
     # via flytekit
-numpy==1.21.0
+numpy==1.21.2
     # via
     #   matplotlib
     #   opencv-python
     #   pandas
     #   pyarrow
-opencv-python==4.5.2.54
+opencv-python==4.5.3.56
     # via -r requirements.in
-pandas==1.3.0
+pandas==1.3.2
     # via flytekit
-pathspec==0.8.1
+pathspec==0.9.0
     # via scantree
 pillow==8.3.1
     # via matplotlib
@@ -85,21 +89,21 @@ python-dateutil==2.8.1
     #   flytekit
     #   matplotlib
     #   pandas
-python-json-logger==2.0.1
+python-json-logger==2.0.2
     # via flytekit
 pytimeparse==1.1.8
     # via flytekit
 pytz==2018.4
     # via
     #   flytekit
     #   pandas
-regex==2021.7.6
+regex==2021.8.28
     # via docker-image-py
-requests==2.25.1
+requests==2.26.0
     # via
     #   flytekit
     #   responses
-responses==0.13.3
+responses==0.13.4
     # via flytekit
 retry==0.9.2
     # via flytekit
@@ -118,9 +122,7 @@ sortedcontainers==2.4.0
     # via flytekit
 statsd==3.3.0
     # via flytekit
-stringcase==1.2.0
-    # via dataclasses-json
-typing-extensions==3.10.0.0
+typing-extensions==3.10.0.2
     # via typing-inspect
 typing-inspect==0.7.1
     # via dataclasses-json
@@ -129,7 +131,7 @@ urllib3==1.26.6
     #   flytekit
     #   requests
     #   responses
-wheel==0.36.2
+wheel==0.37.0
     # via
     #   -r requirements.in
     #   flytekit

diff --git a/tests/flytekit/unit/common_tests/test_launch_plan.py b/tests/flytekit/unit/common_tests/test_launch_plan.py
@@ -340,7 +340,9 @@ def test_serialize():
         == _identifier.Identifier(_identifier.ResourceType.WORKFLOW, "p", "d", "n", "v").to_flyte_idl()
     )
     assert s.spec.auth_role.assumable_iam_role == "iam_role"
-    assert s.spec.default_inputs.parameters["default_input"].default.scalar.primitive.integer == 5
+    assert len(s.spec.default_inputs.parameters) == 1
+    assert s.spec.default_inputs.parameters[0].name == "default_input"
+    assert s.spec.default_inputs.parameters[0].parameter.default.scalar.primitive.integer == 5
 
 
 def test_promote_from_model():

diff --git a/tests/flytekit/unit/common_tests/test_workflow.py b/tests/flytekit/unit/common_tests/test_workflow.py
@@ -366,8 +366,12 @@ def my_list_task(wf_params, a, b):
     serialized = w.serialize()
     assert isinstance(serialized, _workflow_pb2.WorkflowSpec)
     assert len(serialized.template.nodes) == 6
-    assert len(serialized.template.interface.inputs.variables.keys()) == 2
-    assert len(serialized.template.interface.outputs.variables.keys()) == 2
+    assert len(serialized.template.interface.inputs.variables) == 2
+    assert len(serialized.template.interface.outputs.variables) == 2
+    assert serialized.template.interface.inputs.variables[0].name == "required"
+    assert serialized.template.interface.inputs.variables[1].name == "not_required"
+    assert serialized.template.interface.outputs.variables[0].name == "nested_out"
+    assert serialized.template.interface.outputs.variables[1].name == "scalar_out"
 
 
 def test_workflow_disable_default_launch_plan():