Data plugins yt

.gitignore

@@ -26,5 +26,5 @@ dist
 .python-version
 _build/
 docs/source/generated/
-.pytest-flyte
+.pytest_flyte
 htmlcov

dev-requirements.txt

@@ -89,12 +89,16 @@ dockerpty==0.4.1
     # via docker-compose
 docopt==0.6.2
     # via docker-compose
+docstring-parser==0.9.1
+    # via
+    #   -c requirements.txt
+    #   flytekit
 flake8==3.9.2
     # via
     #   -r dev-requirements.in
     #   flake8-black
     #   flake8-isort
-flake8-black==0.2.2
+flake8-black==0.2.3
     # via -r dev-requirements.in
 flake8-isort==4.0.0
     # via -r dev-requirements.in
@@ -136,7 +140,7 @@ markupsafe==2.0.1
     # via
     #   -c requirements.txt
     #   jinja2
-marshmallow==3.12.2
+marshmallow==3.13.0
     # via
     #   -c requirements.txt
     #   dataclasses-json
@@ -166,7 +170,7 @@ natsort==7.1.1
     # via
     #   -c requirements.txt
     #   flytekit
-numpy==1.21.0
+numpy==1.21.1
     # via
     #   -c requirements.txt
     #   pandas
@@ -183,7 +187,7 @@ paramiko==2.7.2
     # via
     #   -c requirements.txt
     #   docker
-pathspec==0.8.1
+pathspec==0.9.0
     # via
     #   -c requirements.txt
     #   black
@@ -315,6 +319,7 @@ texttable==1.6.4
 toml==0.10.2
     # via
     #   coverage
+    #   flake8-black
     #   mypy
     #   pytest
 tomli==1.0.4

doc-requirements.txt

@@ -16,7 +16,7 @@ appnope==0.1.2
     # via
     #   ipykernel
     #   ipython
-astroid==2.6.2
+astroid==2.6.5
     # via sphinx-autoapi
 async-generator==1.10
     # via nbclient
@@ -39,9 +39,9 @@ black==21.7b0
     # via papermill
 bleach==3.3.1
     # via nbconvert
-boto3==1.18.1
+boto3==1.18.6
     # via sagemaker-training
-botocore==1.21.1
+botocore==1.21.6
     # via
     #   boto3
     #   s3transfer
@@ -70,7 +70,7 @@ css-html-js-minify==2.5.5
     # via sphinx-material
 dataclasses-json==0.5.4
     # via flytekit
-debugpy==1.3.0
+debugpy==1.4.1
     # via ipykernel
 decorator==5.0.9
     # via
@@ -82,9 +82,11 @@ deprecated==1.2.12
     # via flytekit
 dirhash==0.2.1
     # via flytekit
-docker-image-py==0.1.10
+docker-image-py==0.1.11
     # via flytekit
-docutils==0.16
+docstring-parser==0.9.1
+    # via flytekit
+docutils==0.17.1
     # via sphinx
 entrypoints==0.3
     # via
@@ -98,7 +100,7 @@ gevent==21.1.2
     # via sagemaker-training
 greenlet==1.1.0
     # via gevent
-grpcio==1.38.1
+grpcio==1.39.0
     # via
     #   -r doc-requirements.in
     #   flytekit
@@ -112,7 +114,7 @@ importlib-metadata==4.6.1
     # via keyring
 inotify_simple==1.2.1
     # via sagemaker-training
-ipykernel==6.0.2
+ipykernel==6.0.3
     # via flytekit
 ipython==7.25.0
     # via ipykernel
@@ -154,7 +156,7 @@ lxml==4.6.3
     # via sphinx-material
 markupsafe==2.0.1
     # via jinja2
-marshmallow==3.12.2
+marshmallow==3.13.0
     # via
     #   dataclasses-json
     #   marshmallow-enum
@@ -188,7 +190,7 @@ nbformat==5.1.3
     #   papermill
 nest-asyncio==1.5.1
     # via nbclient
-numpy==1.21.0
+numpy==1.21.1
     # via
     #   flytekit
     #   pandas
@@ -199,7 +201,7 @@ packaging==21.0
     # via
     #   bleach
     #   sphinx
-pandas==1.3.0
+pandas==1.3.1
     # via flytekit
 pandocfilters==1.4.3
     # via nbconvert
@@ -209,7 +211,7 @@ paramiko==2.7.2
     # via sagemaker-training
 parso==0.8.2
     # via jedi
-pathspec==0.8.1
+pathspec==0.9.0
     # via
     #   black
     #   scantree
@@ -322,7 +324,7 @@ sortedcontainers==2.4.0
     # via flytekit
 soupsieve==2.2.1
     # via beautifulsoup4
-sphinx==3.5.4
+sphinx==4.1.2
     # via
     #   -r doc-requirements.in
     #   furo
@@ -343,7 +345,7 @@ sphinx-fontawesome==0.0.6
     # via -r doc-requirements.in
 sphinx-gallery==0.9.0
     # via -r doc-requirements.in
-sphinx-material==0.0.32
+sphinx-material==0.0.34
     # via -r doc-requirements.in
 sphinx-prompt==1.4.0
     # via -r doc-requirements.in
@@ -373,7 +375,7 @@ textwrap3==0.9.2
     # via ansiwrap
 thrift==0.13.0
     # via hmsclient
-tomli==1.0.4
+tomli==1.1.0
     # via black
 tornado==6.1
     # via

docs/source/_templates/custom.rst

@@ -11,7 +11,11 @@
 
    .. rubric:: {{ _('Methods') }}
    {% for item in methods %}
+
+   {% if item != '__init__' %}
    .. automethod:: {{ item }}
+   {% endif %}
+
    {%- endfor %}
    {% endif %}
    {% endblock %}

docs/source/conf.py

@@ -52,6 +52,8 @@
 # build the templated autosummary files
 autosummary_generate = True
 
+autodoc_typehints = "description"
+
 # autosectionlabel throws warnings if section names are duplicated.
 # The following tells autosectionlabel to not throw a warning for
 # duplicated section names that are in different documents.

docs/source/design/control_plane.rst

@@ -1,55 +1,100 @@
 .. _design-control-plane:
 
-############################
-Control Plane Objects
-############################
-For those who require programmatic access to the control plane, certain APIs are available through "control plane classes".
+###################################################
+FlyteRemote: A Programmatic Control Plane Interface
+###################################################
 
-.. warning::
+For those who require programmatic access to the control plane, the :mod:`~flytekit.remote` module enables you to perform
+certain operations in a python runtime environment.
 
-    The syntax of this section, while it will continue to work, is subject to change.
+Since this section naturally deals with the control plane, this discussion is only relevant for those who have a Flyte
+backend set up and have access to it (a :std:ref:`local sandbox <flyte:deployment-sandbox>` will suffice as well).
 
-Since this section naturally deals with the control plane, this discussion is only relevant for those who have a Flyte backend set up, and have access to it (a local backend will suffice as well of course). These objects do not rely on the underlying code they represent being locally available.
+***************************
+Create a FlyteRemote Object
+***************************
 
-*******
-Setup
-*******
-Similar to the CLIs, this section requires proper configuration. Please follow the setup guide there.
+The :class:`~flytekit.remote.remote.FlyteRemote` class is the entrypoint for programmatically performing operations in a python
+runtime. There are two ways of creating a remote object.
 
-Unlike the CLI case however, you may need to explicitly target the configuration file like so ::
+**Initialize directly**
 
-    from flytekit.configuration.common import CONFIGURATION_SINGLETON
-    CONFIGURATION_SINGLETON.reset_config("/Users/full/path/to/config")
+.. code-block:: python
+
+    from flytekit.remote import FlyteRemote
+
+    remote = FlyteRemote(
+        default_project="project",
+        default_domain="domain",
+        flyte_admin_url="<url>",
+        insecure=True,
+    )
+
+**Initialize from flyte config**
+
+.. TODO: link documentation to flyte config and environment variables
+
+This will initialize a :class:`~flytekit.remote.remote.FlyteRemote` object from your flyte config file or environment variable
+overrides:
 
-*******
-Classes
-*******
-This is not an exhaustive list of the objects available, but should provide the reader with the wherewithal to further ascertain for herself additional functionality.
+.. code-block:: python
 
-Task
-======
-To fetch a Task ::
+    remote = FlyteRemote.from_config()
 
-    from flytekit.common.tasks.task import SdkTask
-    SdkTask.fetch('flytetester', 'staging', 'recipes.core_basic.task.square', '49b6c6bdbb86e974ffd9875cab1f721ada8066a7')
+*****************************
+Fetching Flyte Admin Entities
+*****************************
 
+.. code-block:: python
 
-Workflow
-========
-To inspect a Workflow ::
+    flyte_task = remote.fetch_task(name="my_task", version="v1")
+    flyte_workflow = remote.fetch_workflow(name="my_workflow", version="v1")
+    flyte_launch_plan = remote.fetch_launch_plan(name="my_launch_plan", version="v1")
 
-    from flytekit.common.workflow import SdkWorkflow
-    wf = SdkWorkflow.fetch('flytetester', 'staging', 'recipes.core_basic.basic_workflow.my_wf', '49b6c6bdbb86e974ffd9875cab1f721ada8066a7')
+******************
+Executing Entities
+******************
 
-WorkflowExecution
-=================
-This class represents one run of a workflow.  The ``execution_name`` used here is just the tail end of the URL you see in your browser when looking at a workflow run.
+You can execute all of these flyte entities, which returns a :class:`~flytekit.remote.workflow_execution.FlyteWorkflowExecution` object.
+For more information on flyte entities, see the See the :ref:`remote flyte entities <remote-flyte-execution-objects>`
+reference.
 
 .. code-block:: python
 
-    from flytekit.common.workflow_execution import SdkWorkflowExecution
+    flyte_entity = ...  # one of FlyteTask, FlyteWorkflow, or FlyteLaunchPlan
+    execution = remote.execute(flyte_entity, inputs={...})
+
+********************************
+Waiting for Execution Completion
+********************************
+
+You can use the :meth:`~flytekit.remote.remote.FlyteRemote.wait` method to synchronously wait for the execution to complete:
+
+.. code-block:: python
+
+    completed_execution = remote.wait(execution)
+
+You can also pass in ``wait=True`` to the :meth:`~flytekit.remote.remote.FlyteRemote.execute` method.
+
+.. code-block:: python
+
+    completed_execution = remote.execute(flyte_entity, inputs={...}, wait=True)
+
+********************
+Syncing Remote State
+********************
+
+Use the :meth:`~flytekit.remote.remote.FlyteRemote.sync` method to sync the entity object's state with the remote state
+
+.. code-block:: python
+
+    synced_execution = remote.sync(execution)
+
 
-    e = SdkWorkflowExecution.fetch('project_name', 'development', 'execution_name')
-    e.sync()
+****************************
+Inspecting Execution Objects
+****************************
 
-As a workflow is made up of nodes (each of which can contain a task, a subworkflow, or a launch plan), a workflow execution is made up of node executions (each of which can contain a task execution, a subworkflow execution, or a launch plan execution).
+At any time you can inspect the inputs, outputs, completion status, error status, and other aspects of a workflow
+execution object. See the :ref:`remote execution objects <remote-flyte-execution-objects>` reference for a list
+of all the available attributes.

docs/source/design/index.rst

@@ -8,7 +8,7 @@ Flytekit is comprised of a handful of different logical components, each discuss
 
 * :ref:`Models Files <design-models>` - These are almost Protobuf generated files.
 * :ref:`Authoring <design-authoring>` - This provides the core Flyte authoring experiences, allowing users to write tasks, workflows, and launch plans.
-* :ref:`Control Plane <design-control-plane>` - The code here allows users to interact with the control plane through Python objecs.
+* :ref:`Control Plane <design-control-plane>` - The code here allows users to interact with the control plane through Python objects.
 * :ref:`Execution <design-execution>` - A small shim layer basically that handles interaction with the Flyte ecosystem at execution time.
 * :ref:`CLIs and Clients <design-clis>` - Command line tools users may find themselves interacting with and the control plane client the CLIs call.
 
@@ -19,6 +19,6 @@ Flytekit is comprised of a handful of different logical components, each discuss
 
    models
    authoring
-   control_plane
+   Control Plane <control_plane>
    execution
    clis