add new control plane classes (flyteorg#425)

* implement new control plane classes Signed-off-by: cosmicBboy <[email protected]> * revert dep changes Signed-off-by: cosmicBboy <[email protected]> * remove unneeded mock integration test files Signed-off-by: cosmicBboy <[email protected]> * remove pytest.ini Signed-off-by: cosmicBboy <[email protected]> * add integration tests to ci, update reqs Signed-off-by: cosmicBboy <[email protected]> * add unit tests Signed-off-by: cosmicBboy <[email protected]> * lint Signed-off-by: cosmicBboy <[email protected]> * address comments @wild-endeavor Signed-off-by: cosmicBboy <[email protected]> Signed-off-by: Max Hoffman <[email protected]>
dolthub · Apr 29, 2021 · e5d0f59 · e5d0f59
1 parent 5f60799
commit e5d0f59
Show file tree

Hide file tree

Showing 5 changed files with 110 additions and 10 deletions.
diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -24,6 +24,9 @@ bcrypt==3.2.0
     #   -c requirements.txt
     #   paramiko
 <<<<<<< HEAD
+<<<<<<< HEAD
+=======
+>>>>>>> add new control plane classes (#425)
 black==20.8b1
 >>>>>>> add new control plane classes (#425)
 =======
@@ -33,6 +36,11 @@ black==21.4b2
     #   -c requirements.txt
     #   -r dev-requirements.in
     #   flake8-black
+<<<<<<< HEAD
+=======
+cached-property==1.5.2
+    # via docker-compose
+>>>>>>> add new control plane classes (#425)
 certifi==2020.12.5
     # via
     #   -c requirements.txt
@@ -62,7 +70,11 @@ cryptography==3.4.7
     # via
     #   -c requirements.txt
     #   paramiko
+<<<<<<< HEAD
 dataclasses-json==0.5.3
+=======
+dataclasses-json==0.5.2
+>>>>>>> add new control plane classes (#425)
     # via
     #   -c requirements.txt
     #   flytekit
@@ -103,7 +115,11 @@ flake8==3.9.1
     #   -r dev-requirements.in
     #   flake8-black
     #   flake8-isort
+<<<<<<< HEAD
 flyteidl==0.18.39
+=======
+flyteidl==0.18.38
+>>>>>>> add new control plane classes (#425)
     # via
     #   -c requirements.txt
     #   flytekit
@@ -118,7 +134,15 @@ idna==2.10
 importlib-metadata==4.0.1
     # via
     #   -c requirements.txt
+<<<<<<< HEAD
     #   keyring
+=======
+    #   flake8
+    #   jsonschema
+    #   keyring
+    #   pluggy
+    #   pytest
+>>>>>>> add new control plane classes (#425)
 iniconfig==1.1.1
     # via pytest
 isort==5.8.0
@@ -221,13 +245,17 @@ pyparsing==2.4.7
     #   -c requirements.txt
     #   packaging
 <<<<<<< HEAD
+<<<<<<< HEAD
+=======
+>>>>>>> add new control plane classes (#425)
 pyrsistent==0.17.3
     # via
     #   -c requirements.txt
     #   jsonschema
 pytest-docker==0.10.1
     # via pytest-flyte
 git+git://github.com/flyteorg/pytest-flyte@main#egg=pytest-flyte
+<<<<<<< HEAD
     # via -r dev-requirements.in
 pytest==6.2.3
     # via
@@ -259,6 +287,35 @@ pyyaml==5.4.1
 pytest==6.2.3
     # via -r dev-requirements.in
 >>>>>>> Sqlalchemy Task (#445)
+=======
+    # via -r dev-requirements.in
+pytest==6.2.3
+    # via
+    #   -r dev-requirements.in
+    #   pytest-docker
+    #   pytest-flyte
+python-dateutil==2.8.1
+    # via
+    #   -c requirements.txt
+    #   croniter
+    #   flytekit
+    #   pandas
+python-dotenv==0.17.0
+    # via docker-compose
+pytimeparse==1.1.8
+    # via
+    #   -c requirements.txt
+    #   flytekit
+pytz==2018.4
+    # via
+    #   -c requirements.txt
+    #   flytekit
+    #   pandas
+pyyaml==5.4.1
+    # via
+    #   -c requirements.txt
+    #   docker-compose
+>>>>>>> add new control plane classes (#425)
 regex==2021.4.4
     # via
     #   -c requirements.txt
@@ -271,7 +328,11 @@ requests==2.25.1
     #   docker-compose
     #   flytekit
     #   responses
+<<<<<<< HEAD
 responses==0.13.3
+=======
+responses==0.13.2
+>>>>>>> add new control plane classes (#425)
     # via
     #   -c requirements.txt
     #   flytekit
@@ -338,12 +399,15 @@ typing-extensions==3.7.4.3
 =======
     #   black
     #   importlib-metadata
+<<<<<<< HEAD
 >>>>>>> add new control plane classes (#425)
 =======
 typing-extensions==3.7.4.3
     # via
     #   -c requirements.txt
 >>>>>>> Fix merge
+=======
+>>>>>>> add new control plane classes (#425)
     #   mypy
     #   typing-inspect
 typing-inspect==0.6.0

diff --git a/doc-requirements.txt b/doc-requirements.txt
@@ -116,6 +116,7 @@ entrypoints==0.3
 <<<<<<< HEAD
 <<<<<<< HEAD
 <<<<<<< HEAD
+<<<<<<< HEAD
 flyteidl==0.18.39
 =======
 flyteidl==0.18.37
@@ -129,6 +130,9 @@ flyteidl==0.18.39
 =======
 flyteidl==0.18.37
 >>>>>>> Sqlalchemy Task (#445)
+=======
+flyteidl==0.18.38
+>>>>>>> add new control plane classes (#425)
     # via flytekit
 furo==2021.4.11b34
     # via -r doc-requirements.in
@@ -148,6 +152,7 @@ imagesize==1.2.0
     # via sphinx
 importlib-metadata==4.0.1
 <<<<<<< HEAD
+<<<<<<< HEAD
 <<<<<<< HEAD
     # via
     #   jsonschema
@@ -157,6 +162,11 @@ importlib-metadata==4.0.1
 >>>>>>> Sqlalchemy Task (#445)
     # via keyring
 >>>>>>> Fix merge
+=======
+    # via
+    #   jsonschema
+    #   keyring
+>>>>>>> add new control plane classes (#425)
 inotify_simple==1.2.1
     # via sagemaker-training
 ipykernel==5.5.3
@@ -458,20 +468,29 @@ traitlets==5.0.5
 =======
 typed-ast==1.4.3
 <<<<<<< HEAD
+<<<<<<< HEAD
 =======
 typed-ast==1.4.3
 >>>>>>> Sqlalchemy Task (#445)
     # via black
 >>>>>>> Sqlalchemy Task (#445)
+=======
+    # via
+    #   astroid
+    #   black
+>>>>>>> add new control plane classes (#425)
 typing-extensions==3.7.4.3
     # via typing-inspect
 =======
     # via
     #   astroid
     #   black
+<<<<<<< HEAD
 typing-extensions==3.7.4.3
     # via
     #   black
+=======
+>>>>>>> add new control plane classes (#425)
     #   importlib-metadata
     #   typing-inspect
 >>>>>>> add new control plane classes (#425)

diff --git a/flytekit/control_plane/tasks/executions.py b/flytekit/control_plane/tasks/executions.py
@@ -1,11 +1,11 @@
 from typing import Any, Dict, Optional
 
-from flytekit.clients.helpers import iterate_node_executions as _iterate_node_executions
 from flytekit.common import utils as _common_utils
 from flytekit.common.exceptions import user as _user_exceptions
 from flytekit.common.mixins import artifact as _artifact_mixin
+from flytekit.core.context_manager import FlyteContext
+from flytekit.core.type_engine import TypeEngine
 from flytekit.engines.flyte import engine as _flyte_engine
-from flytekit.models import literals as _literal_models
 from flytekit.models.admin import task_execution as _task_execution_model
 from flytekit.models.core import execution as _execution_models
 
@@ -36,7 +36,7 @@ def inputs(self) -> Dict[str, Any]:
             execution_data = client.get_task_execution_data(self.id)
 
             # Inputs are returned inline unless they are too big, in which case a url blob pointing to them is returned.
-            input_map = _literal_models.LiteralMap({})
+            input_map: _literal_models.LiteralMap = _literal_models.LiteralMap({})
             if bool(execution_data.full_inputs.literals):
                 input_map = execution_data.full_inputs
             elif execution_data.inputs.bytes > 0:
@@ -47,9 +47,7 @@ def inputs(self) -> Dict[str, Any]:
                         _common_utils.load_proto_from_file(_literals_pb2.LiteralMap, tmp_name)
                     )
 
-            # TODO: need to convert flyte literals to python types. For now just use literals
-            # self._inputs = TypeEngine.literal_map_to_kwargs(ctx=FlyteContext.current_context(), lm=input_map)
-            self._inputs = input_map
+            self._inputs = TypeEngine.literal_map_to_kwargs(ctx=FlyteContext.current_context(), lm=input_map)
         return self._inputs
 
     @property
@@ -72,20 +70,19 @@ def outputs(self) -> Dict[str, Any]:
             execution_data = client.get_task_execution_data(self.id)
 
             # Inputs are returned inline unless they are too big, in which case a url blob pointing to them is returned.
-            output_map = _literal_models.LiteralMap({})
             if bool(execution_data.full_outputs.literals):
                 output_map = execution_data.full_outputs
+
             elif execution_data.outputs.bytes > 0:
                 with _common_utils.AutoDeletingTempDir() as t:
                     tmp_name = _os.path.join(t.name, "outputs.pb")
                     _data_proxy.Data.get_data(execution_data.outputs.url, tmp_name)
                     output_map = _literal_models.LiteralMap.from_flyte_idl(
                         _common_utils.load_proto_from_file(_literals_pb2.LiteralMap, tmp_name)
                     )
+                output_map = _literal_models.LiteralMap({})
 
-            # TODO: need to convert flyte literals to python types. For now just use literals
-            # self._outputs = TypeEngine.literal_map_to_kwargs(ctx=FlyteContext.current_context(), lm=output_map)
-            self._outputs = output_map
+            self._outputs = TypeEngine.literal_map_to_kwargs(ctx=FlyteContext.current_context(), lm=output_map)
         return self._outputs
 
     @property

diff --git a/requirements-spark2.txt b/requirements-spark2.txt
@@ -98,6 +98,7 @@ entrypoints==0.3
 <<<<<<< HEAD
 <<<<<<< HEAD
 <<<<<<< HEAD
+<<<<<<< HEAD
 flyteidl==0.18.39
 =======
 flyteidl==0.18.37
@@ -111,6 +112,9 @@ flyteidl==0.18.39
 =======
 flyteidl==0.18.37
 >>>>>>> Sqlalchemy Task (#445)
+=======
+flyteidl==0.18.38
+>>>>>>> add new control plane classes (#425)
     # via flytekit
 gevent==21.1.2
     # via sagemaker-training
@@ -124,6 +128,7 @@ idna==2.10
     # via requests
 importlib-metadata==4.0.1
 <<<<<<< HEAD
+<<<<<<< HEAD
 <<<<<<< HEAD
     # via
     #   jsonschema
@@ -133,6 +138,11 @@ importlib-metadata==4.0.1
 >>>>>>> Sqlalchemy Task (#445)
     # via keyring
 >>>>>>> Fix merge
+=======
+    # via
+    #   jsonschema
+    #   keyring
+>>>>>>> add new control plane classes (#425)
 inotify_simple==1.2.1
     # via sagemaker-training
 ipykernel==5.5.3

diff --git a/requirements.txt b/requirements.txt
@@ -89,6 +89,7 @@ entrypoints==0.3
 <<<<<<< HEAD
 <<<<<<< HEAD
 <<<<<<< HEAD
+<<<<<<< HEAD
 flyteidl==0.18.39
 =======
 flyteidl==0.18.37
@@ -102,6 +103,9 @@ flyteidl==0.18.39
 =======
 flyteidl==0.18.37
 >>>>>>> Sqlalchemy Task (#445)
+=======
+flyteidl==0.18.38
+>>>>>>> add new control plane classes (#425)
     # via flytekit
 gevent==21.1.2
     # via sagemaker-training
@@ -115,6 +119,7 @@ idna==2.10
     # via requests
 importlib-metadata==4.0.1
 <<<<<<< HEAD
+<<<<<<< HEAD
 <<<<<<< HEAD
     # via
     #   jsonschema
@@ -124,6 +129,11 @@ importlib-metadata==4.0.1
 >>>>>>> Sqlalchemy Task (#445)
     # via keyring
 >>>>>>> Fix merge
+=======
+    # via
+    #   jsonschema
+    #   keyring
+>>>>>>> add new control plane classes (#425)
 inotify_simple==1.2.1
     # via sagemaker-training
 ipykernel==5.5.3