Added python pickle docs (flyteorg#477)

Signed-off-by: Kevin Su <[email protected]>

cookbook/common/requirements-common.in

@@ -1,3 +1,3 @@
-flytekit>=0.20.2
+flytekit>=0.24.0b1
 wheel
 matplotlib

cookbook/core/requirements.txt

@@ -4,84 +4,82 @@
 #
 #    /Library/Developer/CommandLineTools/usr/bin/make requirements.txt
 #
-attrs==21.2.0
-    # via scantree
-certifi==2021.5.30
+certifi==2021.10.8
     # via requests
-charset-normalizer==2.0.6
+charset-normalizer==2.0.7
     # via requests
+checksumdir==1.2.0
+    # via flytekit
 click==7.1.2
     # via flytekit
+cloudpickle==2.0.0
+    # via flytekit
 croniter==1.0.15
     # via flytekit
-cycler==0.10.0
+cycler==0.11.0
     # via matplotlib
 dataclasses-json==0.5.6
     # via flytekit
 decorator==5.1.0
     # via retry
 deprecated==1.2.13
     # via flytekit
-dirhash==0.2.1
-    # via flytekit
 diskcache==5.2.1
     # via flytekit
 docker-image-py==0.1.12
     # via flytekit
-docstring-parser==0.11
+docstring-parser==0.12
     # via flytekit
-flyteidl==0.20.2
+flyteidl==0.21.8
     # via flytekit
-flytekit==0.22.2
+flytekit==0.24.0b1
     # via -r ../common/requirements-common.in
-grpcio==1.41.0
+grpcio==1.41.1
     # via flytekit
-idna==3.2
+idna==3.3
     # via requests
 importlib-metadata==4.8.1
     # via keyring
 keyring==23.2.1
     # via flytekit
 kiwisolver==1.3.2
     # via matplotlib
-marshmallow==3.13.0
+marshmallow==3.14.0
     # via
     #   dataclasses-json
     #   marshmallow-enum
     #   marshmallow-jsonschema
 marshmallow-enum==1.5.1
     # via dataclasses-json
-marshmallow-jsonschema==0.12.0
+marshmallow-jsonschema==0.13.0
     # via flytekit
 matplotlib==3.4.3
     # via -r ../common/requirements-common.in
 mypy-extensions==0.4.3
     # via typing-inspect
-natsort==7.1.1
+natsort==8.0.0
     # via flytekit
-numpy==1.21.2
+numpy==1.21.4
     # via
     #   matplotlib
     #   opencv-python
     #   pandas
     #   pyarrow
-opencv-python==4.5.3.56
+opencv-python==4.5.4.58
     # via -r requirements.in
-pandas==1.3.3
+pandas==1.3.4
     # via flytekit
-pathspec==0.9.0
-    # via scantree
-pillow==8.3.2
+pillow==8.4.0
     # via matplotlib
-protobuf==3.18.1
+protobuf==3.19.1
     # via
     #   flyteidl
     #   flytekit
-py==1.10.0
+py==1.11.0
     # via retry
 pyarrow==3.0.0
     # via flytekit
-pyparsing==2.4.7
+pyparsing==3.0.4
     # via matplotlib
 python-dateutil==2.8.1
     # via
@@ -97,26 +95,22 @@ pytz==2018.4
     # via
     #   flytekit
     #   pandas
-regex==2021.9.30
+regex==2021.11.2
     # via docker-image-py
 requests==2.26.0
     # via
     #   flytekit
     #   responses
-responses==0.14.0
+responses==0.15.0
     # via flytekit
 retry==0.9.2
     # via flytekit
-scantree==0.0.1
-    # via dirhash
 six==1.16.0
     # via
-    #   cycler
     #   flytekit
     #   grpcio
     #   python-dateutil
     #   responses
-    #   scantree
 sortedcontainers==2.4.0
     # via flytekit
 statsd==3.3.0
@@ -134,7 +128,7 @@ wheel==0.37.0
     # via
     #   -r ../common/requirements-common.in
     #   flytekit
-wrapt==1.13.1
+wrapt==1.13.3
     # via
     #   deprecated
     #   flytekit

cookbook/core/type_system/custom_objects.py

@@ -7,7 +7,7 @@
 @dataclasses_json are supported.
 
 This example shows how users can serialize custom JSON'able dataclasses between successive tasks using the excellent
-`dataclasses_json <https://pypi.org/project/dataclasses-json/>` library
+`dataclasses_json <https://pypi.org/project/dataclasses-json/>`_ library
 """
 import typing
 from dataclasses import dataclass

cookbook/core/type_system/flyte_pickle.py

@@ -0,0 +1,57 @@
+"""
+.. _flyte_pickle:
+
+Using Flyte Pickle
+----------------------------
+
+Flyte enforces type safety by leveraging type information to be able to compile
+tasks/workflows, which enables all sorts of nice features (like static analysis of tasks/workflows, conditional branching, etc.)
+
+However, we do also want to provide enough flexibility to end-users so that they don't have to put in a lot of up front
+investment figuring out all the types of their data structures before experiencing the value that flyte has to offer.
+
+Flyte supports FlytePickle transformer which will convert any unrecognized type in type hint to
+FlytePickle, and serialize / deserialize the python value to / from a pickle file.
+
+Caveats
+=======
+Pickle can only be used to send objects between the exact same version of Python,
+and we strongly recommend to use python type that flyte support or register a custom transformer
+
+This example shows how users can custom object without register a transformer.
+"""
+from flytekit import task, workflow
+
+
+# %%
+# This People is a user defined complex type, which can be used to pass complex data between tasks.
+# We will serialize this class to a pickle file and pass it between different tasks.
+#
+# .. Note::
+#
+#   Here we can also `turn this object to dataclass <custom_objects.html>`_ to have better performance.
+#   We use simple object here for demo purpose.
+#   You may have some object that can't turn into a dataclass, e.g. NumPy, Tensor.
+class People:
+    def __init__(self, name):
+        self.name = name
+
+
+# %%
+# Object can be returned as outputs or accepted as inputs
+@task
+def greet(name: str) -> People:
+    return People(name)
+
+
+@workflow
+def welcome(name: str) -> People:
+    return greet(name=name)
+
+
+if __name__ == "__main__":
+    """
+    This workflow can be run locally. During local execution also,
+    the custom object (People) will be marshalled to and from python pickle.
+    """
+    welcome(name='Foo')

cookbook/core/type_system/flyte_python_types.py

@@ -88,7 +88,7 @@
    * - User defined types
      - Any
      - Custom Transformers
-     - Use python 3 type hints, but need to provide custom transformers. Refer to :ref:`advanced_custom_types`.
+     - Use python 3 type hints. We use `FlytePickle transformer <flyte_pickle.html>`_ by default, but users still can provide custom transformers. Refer to :ref:`advanced_custom_types`.
 
 
 .. prompt:: bash