Add cache_serialize parameter to tasks (flyteorg#673)

* added cache_serialize parameter for basic tasks Signed-off-by: Daniel Rammer <[email protected]> * fixed typo Signed-off-by: Daniel Rammer <[email protected]> * updated flyteidl version Signed-off-by: Daniel Rammer <[email protected]> * changed flyteidl requirements everywhere Signed-off-by: Daniel Rammer <[email protected]> * remove flyteidl version requires in setup.py so we can use develop Signed-off-by: Daniel Rammer <[email protected]> * removed flyteidl git repos from a variety of requirements packages Signed-off-by: Daniel Rammer <[email protected]> * updated variable discovery_serializable to cache_serializable Signed-off-by: Daniel Rammer <[email protected]> * updated requirements Signed-off-by: Daniel Rammer <[email protected]> * fixed TaskMetadata _cache_serializable variable name Signed-off-by: Daniel Rammer <[email protected]> * propgating cache_serialize parameter through to tasks Signed-off-by: Daniel Rammer <[email protected]> * added cache_serializable to SdkRawContainerTask Signed-off-by: Daniel Rammer <[email protected]> * fixing cache_serializable variable propogation issues Signed-off-by: Daniel Rammer <[email protected]> * added documentation Signed-off-by: Daniel Rammer <[email protected]> * added unit tests for cache_serialize metadata Signed-off-by: Daniel Rammer <[email protected]> * linter added spaces in unit tests Signed-off-by: Daniel Rammer <[email protected]> Signed-off-by: Robert Everson <[email protected]>
reverson · May 27, 2022 · 9a2b375 · 9a2b375
1 parent 4704d83
commit 9a2b375
Show file tree

Hide file tree

Showing 23 changed files with 270 additions and 137 deletions.
diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with python 3.8
+# This file is autogenerated by pip-compile with python 3.9
 # To update, run:
 #
 #    make dev-requirements.txt
@@ -41,7 +41,7 @@ chardet==4.0.0
     # via
     #   -c requirements.txt
     #   binaryornot
-charset-normalizer==2.0.7
+charset-normalizer==2.0.8
     # via
     #   -c requirements.txt
     #   requests
@@ -64,13 +64,13 @@ cookiecutter==1.7.3
     # via
     #   -c requirements.txt
     #   flytekit
-coverage[toml]==6.1.1
+coverage[toml]==6.2
     # via -r dev-requirements.in
 croniter==1.0.15
     # via
     #   -c requirements.txt
     #   flytekit
-cryptography==35.0.0
+cryptography==36.0.0
     # via
     #   -c requirements.txt
     #   paramiko
@@ -87,7 +87,7 @@ deprecated==1.2.13
     # via
     #   -c requirements.txt
     #   flytekit
-diskcache==5.2.1
+diskcache==5.3.0
     # via
     #   -c requirements.txt
     #   flytekit
@@ -105,21 +105,21 @@ dockerpty==0.4.1
     # via docker-compose
 docopt==0.6.2
     # via docker-compose
-docstring-parser==0.12
+docstring-parser==0.13
     # via
     #   -c requirements.txt
     #   flytekit
-filelock==3.3.2
+filelock==3.4.0
     # via virtualenv
-flyteidl==0.21.8
+flyteidl==0.21.11
     # via
     #   -c requirements.txt
     #   flytekit
-grpcio==1.41.1
+grpcio==1.42.0
     # via
     #   -c requirements.txt
     #   flytekit
-identify==2.3.5
+identify==2.4.0
     # via pre-commit
 idna==3.3
     # via
@@ -151,15 +151,15 @@ jsonschema==3.2.0
     # via
     #   -c requirements.txt
     #   docker-compose
-keyring==23.2.1
+keyring==23.4.0
     # via
     #   -c requirements.txt
     #   flytekit
 markupsafe==2.0.1
     # via
     #   -c requirements.txt
     #   jinja2
-marshmallow==3.14.0
+marshmallow==3.14.1
     # via
     #   -c requirements.txt
     #   dataclasses-json
@@ -193,14 +193,14 @@ numpy==1.21.4
     #   -c requirements.txt
     #   pandas
     #   pyarrow
-packaging==21.2
+packaging==21.3
     # via
     #   -c requirements.txt
 pandas==1.3.4
     # via
     #   -c requirements.txt
     #   flytekit
-paramiko==2.8.0
+paramiko==2.8.1
     # via
     #   -c requirements.txt
     #   docker
@@ -214,7 +214,7 @@ poyo==0.5.0
     # via
     #   -c requirements.txt
     #   cookiecutter
-pre-commit==2.15.0
+pre-commit==2.16.0
     # via -r dev-requirements.in
 protobuf==3.19.1
     # via
@@ -225,7 +225,7 @@ py==1.11.0
     # via
     #   -c requirements.txt
     #   retry
-pyarrow==6.0.0
+pyarrow==6.0.1
     # via
     #   -c requirements.txt
     #   flytekit
@@ -237,7 +237,7 @@ pynacl==1.4.0
     # via
     #   -c requirements.txt
     #   paramiko
-pyparsing==2.4.7
+pyparsing==3.0.6
     # via
     #   -c requirements.txt
     #   packaging
@@ -255,7 +255,7 @@ python-dateutil==2.8.1
     #   croniter
     #   flytekit
     #   pandas
-python-dotenv==0.19.1
+python-dotenv==0.19.2
     # via docker-compose
 python-json-logger==2.0.2
     # via
@@ -279,7 +279,7 @@ pyyaml==5.4.1
     #   -c requirements.txt
     #   docker-compose
     #   pre-commit
-regex==2021.11.9
+regex==2021.11.10
     # via
     #   -c requirements.txt
     #   docker-image-py
@@ -291,7 +291,7 @@ requests==2.26.0
     #   docker-compose
     #   flytekit
     #   responses
-responses==0.15.0
+responses==0.16.0
     # via
     #   -c requirements.txt
     #   flytekit
@@ -339,7 +339,7 @@ tomli==1.2.2
     # via
     #   -c requirements.txt
     #   coverage
-typing-extensions==3.10.0.2
+typing-extensions==4.0.1
     # via
     #   -c requirements.txt
     #   mypy