Skip to content

Commit

Permalink
Revert "Adopt flyteidl's ordered variable map change (#608)"
Browse files Browse the repository at this point in the history
This reverts commit 204b13f
Signed-off-by: Sean Lin <[email protected]>
  • Loading branch information
mayitbeegh committed Sep 9, 2021
1 parent 320daff commit 7cfef05
Show file tree
Hide file tree
Showing 12 changed files with 53 additions and 77 deletions.
6 changes: 3 additions & 3 deletions dev-requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ distlib==0.3.2
# via virtualenv
distro==1.6.0
# via docker-compose
docker[ssh]==5.0.1
docker[ssh]==5.0.0
# via docker-compose
docker-compose==1.29.2
# via
Expand All @@ -97,7 +97,7 @@ docstring-parser==0.10
# flytekit
filelock==3.0.12
# via virtualenv
flyteidl==0.20.1
flyteidl==0.19.25
# via
# -c requirements.txt
# flytekit
Expand Down Expand Up @@ -183,7 +183,7 @@ pathspec==0.9.0
# via
# -c requirements.txt
# scantree
platformdirs==2.3.0
platformdirs==2.2.0
# via
# -c requirements.txt
# virtualenv
Expand Down
4 changes: 2 additions & 2 deletions doc-requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ entrypoints==0.3
# jupyter-client
# nbconvert
# papermill
flyteidl==0.20.1
flyteidl==0.19.25
# via flytekit
git+git://github.com/flyteorg/furo@main
# via -r doc-requirements.in
Expand Down Expand Up @@ -214,7 +214,7 @@ pexpect==4.8.0
# via ipython
pickleshare==0.7.5
# via ipython
platformdirs==2.3.0
platformdirs==2.2.0
# via black
prompt-toolkit==3.0.20
# via ipython
Expand Down
30 changes: 9 additions & 21 deletions flytekit/models/interface.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import typing
from collections import OrderedDict

import six as _six
from flyteidl.core import interface_pb2 as _interface_pb2

from flytekit.models import common as _common
Expand Down Expand Up @@ -73,17 +73,15 @@ def to_flyte_idl(self):
"""
:rtype: dict[Text, Variable]
"""
return _interface_pb2.VariableMap(
variables=[_interface_pb2.VariableMapEntry(name=k, var=v.to_flyte_idl()) for k, v in self.variables.items()]
)
return _interface_pb2.VariableMap(variables={k: v.to_flyte_idl() for k, v in _six.iteritems(self.variables)})

@classmethod
def from_flyte_idl(cls, pb2_object):
"""
:param dict[Text, Variable] pb2_object:
:rtype: VariableMap
"""
return cls(OrderedDict((v.name, Variable.from_flyte_idl(v.var)) for v in pb2_object.variables))
return cls({k: Variable.from_flyte_idl(v) for k, v in _six.iteritems(pb2_object.variables)})


class TypedInterface(_common.FlyteIdlEntity):
Expand All @@ -108,15 +106,9 @@ def outputs(self) -> typing.Dict[str, Variable]:

def to_flyte_idl(self) -> _interface_pb2.TypedInterface:
return _interface_pb2.TypedInterface(
inputs=_interface_pb2.VariableMap(
variables=[
_interface_pb2.VariableMapEntry(name=k, var=v.to_flyte_idl()) for k, v in self.inputs.items()
]
),
inputs=_interface_pb2.VariableMap(variables={k: v.to_flyte_idl() for k, v in _six.iteritems(self.inputs)}),
outputs=_interface_pb2.VariableMap(
variables=[
_interface_pb2.VariableMapEntry(name=k, var=v.to_flyte_idl()) for k, v in self.outputs.items()
]
variables={k: v.to_flyte_idl() for k, v in _six.iteritems(self.outputs)}
),
)

Expand All @@ -126,8 +118,8 @@ def from_flyte_idl(cls, proto: _interface_pb2.TypedInterface) -> "TypedInterface
:param proto:
"""
return cls(
inputs=OrderedDict((v.name, Variable.from_flyte_idl(v.var)) for v in proto.inputs.variables),
outputs=OrderedDict((v.name, Variable.from_flyte_idl(v.var)) for v in proto.outputs.variables),
inputs={k: Variable.from_flyte_idl(v) for k, v in _six.iteritems(proto.inputs.variables)},
outputs={k: Variable.from_flyte_idl(v) for k, v in _six.iteritems(proto.outputs.variables)},
)


Expand Down Expand Up @@ -219,9 +211,7 @@ def to_flyte_idl(self):
:rtype: flyteidl.core.interface_pb2.ParameterMap
"""
return _interface_pb2.ParameterMap(
parameters=[
_interface_pb2.ParameterMapEntry(name=k, parameter=v.to_flyte_idl()) for k, v in self.parameters.items()
]
parameters={k: v.to_flyte_idl() for k, v in _six.iteritems(self.parameters)},
)

@classmethod
Expand All @@ -230,6 +220,4 @@ def from_flyte_idl(cls, pb2_object):
:param flyteidl.core.interface_pb2.ParameterMap pb2_object:
:rtype: ParameterMap
"""
return cls(
parameters=OrderedDict((v.name, Parameter.from_flyte_idl(v.parameter)) for v in pb2_object.parameters)
)
return cls(parameters={k: Parameter.from_flyte_idl(v) for k, v in _six.iteritems(pb2_object.parameters)})
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
flytekit>=0.22.0b6
flytekit>=0.20.1
sqlalchemy
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#
# This file is autogenerated by pip-compile with python 3.9
# This file is autogenerated by pip-compile with python 3.8
# To update, run:
#
# pip-compile requirements.in
Expand All @@ -26,15 +26,11 @@ deprecated==1.2.12
# via flytekit
dirhash==0.2.1
# via flytekit
diskcache==5.2.1
# via flytekit
docker-image-py==0.1.10
# via flytekit
docstring-parser==0.10
# via flytekit
flyteidl==0.20.1
flyteidl==0.19.5
# via flytekit
flytekit==0.22.0b6
flytekit==0.20.1
# via -r requirements.in
greenlet==1.1.0
# via sqlalchemy
Expand Down
4 changes: 2 additions & 2 deletions requirements-spark2.txt
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ entrypoints==0.3
# jupyter-client
# nbconvert
# papermill
flyteidl==0.20.1
flyteidl==0.19.25
# via flytekit
gevent==21.8.0
# via sagemaker-training
Expand Down Expand Up @@ -183,7 +183,7 @@ pexpect==4.8.0
# via ipython
pickleshare==0.7.5
# via ipython
platformdirs==2.3.0
platformdirs==2.2.0
# via black
prompt-toolkit==3.0.20
# via ipython
Expand Down
4 changes: 2 additions & 2 deletions requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ entrypoints==0.3
# jupyter-client
# nbconvert
# papermill
flyteidl==0.20.1
flyteidl==0.19.25
# via flytekit
gevent==21.8.0
# via sagemaker-training
Expand Down Expand Up @@ -183,7 +183,7 @@ pexpect==4.8.0
# via ipython
pickleshare==0.7.5
# via ipython
platformdirs==2.3.0
platformdirs==2.2.0
# via black
prompt-toolkit==3.0.20
# via ipython
Expand Down
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@
]
},
install_requires=[
"flyteidl>=0.20.1,<0.21.0",
"flyteidl>=0.19.18,<0.20.0",
"wheel>=0.30.0,<1.0.0",
"pandas>=1.0.0,<2.0.0",
"pyarrow>=2.0.0,<4.0.0",
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
flytekit>=0.22.0b6
flytekit>=0.17.0b0
wheel
matplotlib
opencv-python
Original file line number Diff line number Diff line change
Expand Up @@ -8,43 +8,39 @@ attrs==21.2.0
# via scantree
certifi==2021.5.30
# via requests
charset-normalizer==2.0.4
chardet==4.0.0
# via requests
click==7.1.2
# via flytekit
croniter==1.0.15
# via flytekit
cycler==0.10.0
# via matplotlib
dataclasses-json==0.5.5
dataclasses-json==0.5.4
# via flytekit
decorator==5.0.9
# via retry
deprecated==1.2.13
deprecated==1.2.12
# via flytekit
dirhash==0.2.1
# via flytekit
diskcache==5.2.1
docker-image-py==0.1.10
# via flytekit
docker-image-py==0.1.12
flyteidl==0.19.11
# via flytekit
docstring-parser==0.10
# via flytekit
flyteidl==0.20.2
# via flytekit
flytekit==0.22.1
flytekit==0.20.1
# via -r requirements.in
grpcio==1.39.0
grpcio==1.38.1
# via flytekit
idna==3.2
idna==2.10
# via requests
importlib-metadata==4.8.1
importlib-metadata==4.6.1
# via keyring
keyring==23.1.0
keyring==23.0.1
# via flytekit
kiwisolver==1.3.2
kiwisolver==1.3.1
# via matplotlib
marshmallow==3.13.0
marshmallow==3.12.2
# via
# dataclasses-json
# marshmallow-enum
Expand All @@ -53,25 +49,25 @@ marshmallow-enum==1.5.1
# via dataclasses-json
marshmallow-jsonschema==0.12.0
# via flytekit
matplotlib==3.4.3
matplotlib==3.4.2
# via -r requirements.in
mypy-extensions==0.4.3
# via typing-inspect
natsort==7.1.1
# via flytekit
numpy==1.21.2
numpy==1.21.0
# via
# matplotlib
# opencv-python
# pandas
# pyarrow
opencv-python==4.5.3.56
opencv-python==4.5.2.54
# via -r requirements.in
pandas==1.3.2
pandas==1.3.0
# via flytekit
pathspec==0.9.0
pathspec==0.8.1
# via scantree
pillow==8.3.2
pillow==8.3.1
# via matplotlib
protobuf==3.17.3
# via
Expand All @@ -89,21 +85,21 @@ python-dateutil==2.8.1
# flytekit
# matplotlib
# pandas
python-json-logger==2.0.2
python-json-logger==2.0.1
# via flytekit
pytimeparse==1.1.8
# via flytekit
pytz==2018.4
# via
# flytekit
# pandas
regex==2021.8.28
regex==2021.7.6
# via docker-image-py
requests==2.26.0
requests==2.25.1
# via
# flytekit
# responses
responses==0.13.4
responses==0.13.3
# via flytekit
retry==0.9.2
# via flytekit
Expand All @@ -122,7 +118,9 @@ sortedcontainers==2.4.0
# via flytekit
statsd==3.3.0
# via flytekit
typing-extensions==3.10.0.2
stringcase==1.2.0
# via dataclasses-json
typing-extensions==3.10.0.0
# via typing-inspect
typing-inspect==0.7.1
# via dataclasses-json
Expand All @@ -131,7 +129,7 @@ urllib3==1.26.6
# flytekit
# requests
# responses
wheel==0.37.0
wheel==0.36.2
# via
# -r requirements.in
# flytekit
Expand Down
4 changes: 1 addition & 3 deletions tests/flytekit/unit/common_tests/test_launch_plan.py
Original file line number Diff line number Diff line change
Expand Up @@ -340,9 +340,7 @@ def test_serialize():
== _identifier.Identifier(_identifier.ResourceType.WORKFLOW, "p", "d", "n", "v").to_flyte_idl()
)
assert s.spec.auth_role.assumable_iam_role == "iam_role"
assert len(s.spec.default_inputs.parameters) == 1
assert s.spec.default_inputs.parameters[0].name == "default_input"
assert s.spec.default_inputs.parameters[0].parameter.default.scalar.primitive.integer == 5
assert s.spec.default_inputs.parameters["default_input"].default.scalar.primitive.integer == 5


def test_promote_from_model():
Expand Down
8 changes: 2 additions & 6 deletions tests/flytekit/unit/common_tests/test_workflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -366,12 +366,8 @@ def my_list_task(wf_params, a, b):
serialized = w.serialize()
assert isinstance(serialized, _workflow_pb2.WorkflowSpec)
assert len(serialized.template.nodes) == 6
assert len(serialized.template.interface.inputs.variables) == 2
assert len(serialized.template.interface.outputs.variables) == 2
assert serialized.template.interface.inputs.variables[0].name == "required"
assert serialized.template.interface.inputs.variables[1].name == "not_required"
assert serialized.template.interface.outputs.variables[0].name == "nested_out"
assert serialized.template.interface.outputs.variables[1].name == "scalar_out"
assert len(serialized.template.interface.inputs.variables.keys()) == 2
assert len(serialized.template.interface.outputs.variables.keys()) == 2


def test_workflow_disable_default_launch_plan():
Expand Down

0 comments on commit 7cfef05

Please sign in to comment.