diff --git a/dev-requirements.txt b/dev-requirements.txt index 02707a107a..11849ca345 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.8 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # make dev-requirements.txt @@ -41,7 +41,7 @@ chardet==4.0.0 # via # -c requirements.txt # binaryornot -charset-normalizer==2.0.7 +charset-normalizer==2.0.8 # via # -c requirements.txt # requests @@ -64,13 +64,13 @@ cookiecutter==1.7.3 # via # -c requirements.txt # flytekit -coverage[toml]==6.1.1 +coverage[toml]==6.2 # via -r dev-requirements.in croniter==1.0.15 # via # -c requirements.txt # flytekit -cryptography==35.0.0 +cryptography==36.0.0 # via # -c requirements.txt # paramiko @@ -87,7 +87,7 @@ deprecated==1.2.13 # via # -c requirements.txt # flytekit -diskcache==5.2.1 +diskcache==5.3.0 # via # -c requirements.txt # flytekit @@ -105,21 +105,21 @@ dockerpty==0.4.1 # via docker-compose docopt==0.6.2 # via docker-compose -docstring-parser==0.12 +docstring-parser==0.13 # via # -c requirements.txt # flytekit -filelock==3.3.2 +filelock==3.4.0 # via virtualenv -flyteidl==0.21.8 +flyteidl==0.21.11 # via # -c requirements.txt # flytekit -grpcio==1.41.1 +grpcio==1.42.0 # via # -c requirements.txt # flytekit -identify==2.3.5 +identify==2.4.0 # via pre-commit idna==3.3 # via @@ -151,7 +151,7 @@ jsonschema==3.2.0 # via # -c requirements.txt # docker-compose -keyring==23.2.1 +keyring==23.4.0 # via # -c requirements.txt # flytekit @@ -159,7 +159,7 @@ markupsafe==2.0.1 # via # -c requirements.txt # jinja2 -marshmallow==3.14.0 +marshmallow==3.14.1 # via # -c requirements.txt # dataclasses-json @@ -193,14 +193,14 @@ numpy==1.21.4 # -c requirements.txt # pandas # pyarrow -packaging==21.2 +packaging==21.3 # via # -c requirements.txt pandas==1.3.4 # via # -c requirements.txt # flytekit -paramiko==2.8.0 +paramiko==2.8.1 # via # -c requirements.txt # docker @@ -214,7 +214,7 @@ poyo==0.5.0 # via # -c requirements.txt # cookiecutter -pre-commit==2.15.0 +pre-commit==2.16.0 # via -r dev-requirements.in protobuf==3.19.1 # via @@ -225,7 +225,7 @@ py==1.11.0 # via # -c requirements.txt # retry -pyarrow==6.0.0 +pyarrow==6.0.1 # via # -c requirements.txt # flytekit @@ -237,7 +237,7 @@ pynacl==1.4.0 # via # -c requirements.txt # paramiko -pyparsing==2.4.7 +pyparsing==3.0.6 # via # -c requirements.txt # packaging @@ -255,7 +255,7 @@ python-dateutil==2.8.1 # croniter # flytekit # pandas -python-dotenv==0.19.1 +python-dotenv==0.19.2 # via docker-compose python-json-logger==2.0.2 # via @@ -279,7 +279,7 @@ pyyaml==5.4.1 # -c requirements.txt # docker-compose # pre-commit -regex==2021.11.9 +regex==2021.11.10 # via # -c requirements.txt # docker-image-py @@ -291,7 +291,7 @@ requests==2.26.0 # docker-compose # flytekit # responses -responses==0.15.0 +responses==0.16.0 # via # -c requirements.txt # flytekit @@ -339,7 +339,7 @@ tomli==1.2.2 # via # -c requirements.txt # coverage -typing-extensions==3.10.0.2 +typing-extensions==4.0.1 # via # -c requirements.txt # mypy diff --git a/doc-requirements.txt b/doc-requirements.txt index c8fd010b7b..9f71ca88c0 100644 --- a/doc-requirements.txt +++ b/doc-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.8 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # make doc-requirements.txt @@ -12,7 +12,7 @@ ansiwrap==0.8.4 # via papermill arrow==1.2.1 # via jinja2-time -astroid==2.8.4 +astroid==2.9.0 # via sphinx-autoapi attrs==21.2.0 # via jsonschema @@ -29,13 +29,13 @@ beautifulsoup4==4.10.0 # sphinx-material binaryornot==0.4.4 # via cookiecutter -black==21.10b0 +black==21.11b1 # via papermill bleach==4.1.0 # via nbconvert -boto3==1.20.2 +boto3==1.20.17 # via sagemaker-training -botocore==1.23.2 +botocore==1.23.17 # via # boto3 # s3transfer @@ -48,7 +48,7 @@ cffi==1.15.0 # pynacl chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.7 +charset-normalizer==2.0.8 # via requests checksumdir==1.2.0 # via flytekit @@ -65,7 +65,7 @@ cookiecutter==1.7.3 # via flytekit croniter==1.0.15 # via flytekit -cryptography==35.0.0 +cryptography==36.0.0 # via # -r doc-requirements.in # paramiko @@ -82,11 +82,11 @@ defusedxml==0.7.1 # via nbconvert deprecated==1.2.13 # via flytekit -diskcache==5.2.1 +diskcache==5.3.0 # via flytekit docker-image-py==0.1.12 # via flytekit -docstring-parser==0.12 +docstring-parser==0.13 # via flytekit docutils==0.17.1 # via sphinx @@ -95,7 +95,7 @@ entrypoints==0.3 # jupyter-client # nbconvert # papermill -flyteidl==0.21.8 +flyteidl==0.21.11 # via flytekit furo @ git+https://github.com/flyteorg/furo@main # via -r doc-requirements.in @@ -103,7 +103,7 @@ gevent==21.8.0 # via sagemaker-training greenlet==1.1.2 # via gevent -grpcio==1.41.1 +grpcio==1.42.0 # via # -r doc-requirements.in # flytekit @@ -115,19 +115,17 @@ imagesize==1.3.0 # via sphinx importlib-metadata==4.8.2 # via keyring -importlib-resources==5.4.0 - # via jsonschema inotify_simple==1.2.1 # via sagemaker-training ipykernel==5.5.6 # via flytekit -ipython==7.29.0 +ipython==7.30.0 # via ipykernel ipython-genutils==0.2.0 # via # ipykernel # nbformat -jedi==0.18.0 +jedi==0.18.1 # via ipython jeepney==0.7.1 # via @@ -148,7 +146,7 @@ jmespath==0.10.0 # botocore jsonschema==4.2.1 # via nbformat -jupyter-client==7.0.6 +jupyter-client==7.1.0 # via # ipykernel # nbclient @@ -161,7 +159,7 @@ jupyterlab-pygments==0.1.2 # via nbconvert k8s-proto==0.0.3 # via flytekit -keyring==23.2.1 +keyring==23.4.0 # via flytekit lazy-object-proxy==1.6.0 # via astroid @@ -169,7 +167,7 @@ lxml==4.6.5 # via sphinx-material markupsafe==2.0.1 # via jinja2 -marshmallow==3.14.0 +marshmallow==3.14.1 # via # dataclasses-json # marshmallow-enum @@ -188,18 +186,18 @@ mypy-extensions==0.4.3 # typing-inspect natsort==8.0.0 # via flytekit -nbclient==0.5.5 +nbclient==0.5.9 # via # nbconvert # papermill -nbconvert==6.2.0 +nbconvert==6.3.0 # via flytekit nbformat==5.1.3 # via # nbclient # nbconvert # papermill -nest-asyncio==1.5.1 +nest-asyncio==1.5.2 # via # jupyter-client # nbclient @@ -210,7 +208,7 @@ numpy==1.21.4 # pyarrow # sagemaker-training # scipy -packaging==21.2 +packaging==21.3 # via # bleach # sphinx @@ -220,9 +218,9 @@ pandocfilters==1.5.0 # via nbconvert papermill==2.3.3 # via flytekit -paramiko==2.8.0 +paramiko==2.8.1 # via sagemaker-training -parso==0.8.2 +parso==0.8.3 # via jedi pathspec==0.9.0 # via black @@ -234,7 +232,7 @@ platformdirs==2.4.0 # via black poyo==0.5.0 # via cookiecutter -prompt-toolkit==3.0.22 +prompt-toolkit==3.0.23 # via ipython protobuf==3.19.1 # via @@ -250,7 +248,7 @@ py==1.11.0 # via retry py4j==0.10.9.2 # via pyspark -pyarrow==6.0.0 +pyarrow==6.0.1 # via flytekit pycparser==2.21 # via cffi @@ -263,7 +261,7 @@ pygments==2.10.0 # sphinx-prompt pynacl==1.4.0 # via paramiko -pyparsing==2.4.7 +pyparsing==3.0.6 # via packaging pyrsistent==0.18.0 # via jsonschema @@ -296,7 +294,7 @@ pyyaml==6.0 # sphinx-autoapi pyzmq==22.3.0 # via jupyter-client -regex==2021.11.9 +regex==2021.11.10 # via # black # docker-image-py @@ -307,7 +305,7 @@ requests==2.26.0 # papermill # responses # sphinx -responses==0.15.0 +responses==0.16.0 # via flytekit retry==0.9.2 # via flytekit @@ -317,7 +315,7 @@ s3transfer==0.5.0 # via boto3 sagemaker-training==3.9.2 # via flytekit -scipy==1.7.2 +scipy==1.7.3 # via sagemaker-training secretstorage==3.3.1 # via keyring @@ -335,13 +333,13 @@ six==1.16.0 # sagemaker-training # sphinx-code-include # thrift -snowballstemmer==2.1.0 +snowballstemmer==2.2.0 # via sphinx sortedcontainers==2.4.0 # via flytekit -soupsieve==2.3 +soupsieve==2.3.1 # via beautifulsoup4 -sphinx==4.2.0 +sphinx==4.3.1 # via # -r doc-requirements.in # furo @@ -361,7 +359,7 @@ sphinx-copybutton==0.4.0 # via -r doc-requirements.in sphinx-fontawesome==0.0.6 # via -r doc-requirements.in -sphinx-gallery==0.10.0 +sphinx-gallery==0.10.1 # via -r doc-requirements.in sphinx-material==0.0.35 # via -r doc-requirements.in @@ -411,7 +409,7 @@ traitlets==5.1.1 # nbclient # nbconvert # nbformat -typing-extensions==3.10.0.2 +typing-extensions==4.0.1 # via # astroid # black @@ -442,9 +440,7 @@ wrapt==1.13.3 # deprecated # flytekit zipp==3.6.0 - # via - # importlib-metadata - # importlib-resources + # via importlib-metadata zope.event==4.5.0 # via gevent zope.interface==5.4.0 diff --git a/flytekit/common/tasks/hive_task.py b/flytekit/common/tasks/hive_task.py index 6a7a6b7d48..0a604d5a1b 100644 --- a/flytekit/common/tasks/hive_task.py +++ b/flytekit/common/tasks/hive_task.py @@ -50,6 +50,7 @@ def __init__( cluster_label, tags, environment, + cache_serializable, ): """ :param task_function: Function container user code. This will be executed via the SDK's engine. @@ -70,6 +71,7 @@ def __init__( :param Text cluster_label: :param list[Text] tags: :param dict[Text, Text] environment: + :param bool cache_serializable: """ self._task_function = task_function super(SdkHiveTask, self).__init__( @@ -91,6 +93,7 @@ def __init__( discoverable, timeout, environment, + cache_serializable, {}, ) self._validate_task_parameters(cluster_label, tags) diff --git a/flytekit/common/tasks/presto_task.py b/flytekit/common/tasks/presto_task.py index 47e198494d..c8f7d300a5 100644 --- a/flytekit/common/tasks/presto_task.py +++ b/flytekit/common/tasks/presto_task.py @@ -35,6 +35,7 @@ def __init__( retries=1, timeout=None, deprecated=None, + cache_serializable=False, ): """ :param Text statement: Presto query specification @@ -49,6 +50,7 @@ def __init__( :param datetime.timedelta timeout: :param Text deprecated: This string can be used to mark the task as deprecated. Consumers of the task will receive deprecation warnings. + :param bool cache_serializable: """ # Set as class fields which are used down below to configure implicit @@ -66,6 +68,7 @@ def __init__( interruptible, discovery_version, deprecated, + cache_serializable, ) presto_query = _presto_models.PrestoQuery( diff --git a/flytekit/common/tasks/raw_container.py b/flytekit/common/tasks/raw_container.py index c290b36205..5168e744f3 100644 --- a/flytekit/common/tasks/raw_container.py +++ b/flytekit/common/tasks/raw_container.py @@ -134,6 +134,7 @@ def __init__( discovery_version: str = None, retries: int = 1, timeout: _datetime.timedelta = None, + cache_serializable: bool = False, ): """ :param inputs: @@ -155,6 +156,7 @@ def __init__( :param discovery_version: :param retries: :param timeout: + :param cache_serializable: :param input_data_dir: This is the directory where data will be downloaded to :param output_data_dir: This is the directory where data will be uploaded from :param metadata_format: Format in which the metadata will be available for the script @@ -183,6 +185,7 @@ def __init__( interruptible, discovery_version, None, + cache_serializable, ) # The interface is defined using the inputs and outputs diff --git a/flytekit/common/tasks/sdk_dynamic.py b/flytekit/common/tasks/sdk_dynamic.py index 2e64051488..469d14085f 100644 --- a/flytekit/common/tasks/sdk_dynamic.py +++ b/flytekit/common/tasks/sdk_dynamic.py @@ -321,6 +321,7 @@ def __init__( allowed_failure_ratio, max_concurrency, environment, + cache_serializable, custom, ): """ @@ -343,6 +344,7 @@ def __init__( :param float allowed_failure_ratio: :param int max_concurrency: :param dict[Text, Text] environment: + :param bool cache_serializable: :param dict[Text, T] custom: """ _sdk_runnable.SdkRunnableTask.__init__( @@ -365,6 +367,7 @@ def __init__( discoverable, timeout, environment, + cache_serializable, custom, ) diff --git a/flytekit/common/tasks/sdk_runnable.py b/flytekit/common/tasks/sdk_runnable.py index 0ca5e3db1a..700cdb649e 100644 --- a/flytekit/common/tasks/sdk_runnable.py +++ b/flytekit/common/tasks/sdk_runnable.py @@ -386,6 +386,7 @@ def __init__( discoverable, timeout, environment, + cache_serializable, custom, ): """ @@ -406,6 +407,7 @@ def __init__( :param bool discoverable: :param datetime.timedelta timeout: :param dict[Text, Text] environment: + :param bool cache_serializable: :param dict[Text, T] custom: """ # Circular dependency @@ -426,6 +428,7 @@ def __init__( interruptible, discovery_version, deprecated, + cache_serializable, ), # TODO: If we end up using SdkRunnableTask for the new code, make sure this is set correctly. _interface.TypedInterface({}, {}), diff --git a/flytekit/common/tasks/sidecar_task.py b/flytekit/common/tasks/sidecar_task.py index d79bf42e93..a5c7f08af4 100644 --- a/flytekit/common/tasks/sidecar_task.py +++ b/flytekit/common/tasks/sidecar_task.py @@ -37,6 +37,7 @@ def __init__( discoverable, timeout, environment, + cache_serializable, pod_spec=None, primary_container_name=None, annotations=None, @@ -74,6 +75,7 @@ def __init__( discoverable, timeout, environment, + cache_serializable, custom=None, ) @@ -183,6 +185,7 @@ def __init__( allowed_failure_ratio, max_concurrency, environment, + cache_serializable, pod_spec=None, primary_container_name=None, annotations=None, @@ -208,6 +211,7 @@ def __init__( :param float allowed_failure_ratio: :param int max_concurrency: :param dict[Text, Text] environment: + :param bool cache_serializable: :param generated_pb2.PodSpec pod_spec: :param Text primary_container_name: :param dict[Text, Text] annotations: @@ -235,6 +239,7 @@ def __init__( discoverable, timeout, environment, + cache_serializable, pod_spec=pod_spec, primary_container_name=primary_container_name, annotations=annotations, diff --git a/flytekit/common/tasks/spark_task.py b/flytekit/common/tasks/spark_task.py index 4b994286c1..fc68479d89 100644 --- a/flytekit/common/tasks/spark_task.py +++ b/flytekit/common/tasks/spark_task.py @@ -81,6 +81,7 @@ def __init__( spark_conf, hadoop_conf, environment, + cache_serializable, ): """ :param task_function: Function container user code. This will be executed via the SDK's engine. @@ -94,6 +95,7 @@ def __init__( :param dict[Text,Text] spark_conf: :param dict[Text,Text] hadoop_conf: :param dict[Text,Text] environment: [optional] environment variables to set when executing this task. + :param bool cache_serializable: """ spark_exec_path = _os.path.abspath(_entrypoint.__file__) @@ -127,6 +129,7 @@ def __init__( discoverable, timeout, environment, + cache_serializable, _MessageToDict(self._spark_job.to_flyte_idl()), ) diff --git a/flytekit/contrib/sensors/task.py b/flytekit/contrib/sensors/task.py index aa60ca8112..6cc2689c65 100644 --- a/flytekit/contrib/sensors/task.py +++ b/flytekit/contrib/sensors/task.py @@ -118,6 +118,7 @@ def wrapper(fn): custom={}, discovery_version="", discoverable=False, + cache_serializable=False, ) # This is syntactic-sugar, so that when calling this decorator without args, you can either diff --git a/flytekit/core/base_task.py b/flytekit/core/base_task.py index 570e1c4d0c..53fa011185 100644 --- a/flytekit/core/base_task.py +++ b/flytekit/core/base_task.py @@ -72,6 +72,7 @@ class TaskMetadata(object): Args: cache (bool): Indicates if caching should be enabled. See :std:ref:`Caching ` + cache_serialize (bool): Indicates if identical (ie. same inputs) instances of this task should be executed in serial when caching is enabled. See :std:ref:`Caching ` cache_version (str): Version to be used for the cached value interruptible (Optional[bool]): Indicates that this task can be interrupted and/or scheduled on nodes with lower QoS guarantees that can include pre-emption. This can reduce the monetary cost executions incur at the @@ -85,6 +86,7 @@ class TaskMetadata(object): """ cache: bool = False + cache_serialize: bool = False cache_version: str = "" interruptible: Optional[bool] = None deprecated: str = "" @@ -99,6 +101,8 @@ def __post_init__(self): raise ValueError("timeout should be duration represented as either a datetime.timedelta or int seconds") if self.cache and not self.cache_version: raise ValueError("Caching is enabled ``cache=True`` but ``cache_version`` is not set.") + if self.cache_serialize and not self.cache: + raise ValueError("Cache serialize is enabled ``cache_serialize=True`` but ``cache`` is not enabled.") @property def retry_strategy(self) -> _literal_models.RetryStrategy: @@ -120,6 +124,7 @@ def to_taskmetadata_model(self) -> _task_model.TaskMetadata: interruptible=self.interruptible, discovery_version=self.cache_version, deprecated_error_message=self.deprecated, + cache_serializable=self.cache_serialize, ) diff --git a/flytekit/core/task.py b/flytekit/core/task.py index 45b7e2dc94..e070359c98 100644 --- a/flytekit/core/task.py +++ b/flytekit/core/task.py @@ -75,6 +75,7 @@ def task( _task_function: Optional[Callable] = None, task_config: Optional[Any] = None, cache: bool = False, + cache_serialize: bool = False, cache_version: str = "", retries: int = 0, interruptible: Optional[bool] = None, @@ -121,6 +122,10 @@ def my_task(x: int, y: typing.Dict[str, str]) -> str: :param task_config: This argument provides configuration for a specific task types. Please refer to the plugins documentation for the right object to use. :param cache: Boolean that indicates if caching should be enabled + :param cache_serialize: Boolean that indicates if identical (ie. same inputs) instances of this task should be + executed in serial when caching is enabled. This means that given multiple concurrent executions over + identical inputs, only a single instance executes and the rest wait to reuse the cached results. This + parameter does nothing without also setting the cache parameter. :param cache_version: Cache version to use. Changes to the task signature will automatically trigger a cache miss, but you can always manually update this field as well to force a cache miss. You should also manually bump this version if the function body/business logic has changed, but the signature hasn't. @@ -176,6 +181,7 @@ def foo2(): def wrapper(fn) -> PythonFunctionTask: _metadata = TaskMetadata( cache=cache, + cache_serialize=cache_serialize, cache_version=cache_version, retries=retries, interruptible=interruptible, diff --git a/flytekit/models/task.py b/flytekit/models/task.py index c621dd9385..23ff26d87d 100644 --- a/flytekit/models/task.py +++ b/flytekit/models/task.py @@ -180,6 +180,7 @@ def __init__( interruptible, discovery_version, deprecated_error_message, + cache_serializable, ): """ Information needed at runtime to determine behavior such as whether or not outputs are discoverable, timeouts, @@ -197,6 +198,8 @@ def __init__( task are the same and the discovery_version is also the same. :param Text deprecated: This string can be used to mark the task as deprecated. Consumers of the task will receive deprecation warnings. + :param bool cache_serializable: Whether or not caching operations are executed in serial. This means only a + single instance over identical inputs is executed, other concurrent executions wait for the cached results. """ self._discoverable = discoverable self._runtime = runtime @@ -205,6 +208,7 @@ def __init__( self._retries = retries self._discovery_version = discovery_version self._deprecated_error_message = deprecated_error_message + self._cache_serializable = cache_serializable @property def discoverable(self): @@ -265,6 +269,15 @@ def deprecated_error_message(self): """ return self._deprecated_error_message + @property + def cache_serializable(self): + """ + Whether or not caching operations are executed in serial. This means only a single instance over identical + inputs is executed, other concurrent executions wait for the cached results. + :rtype: bool + """ + return self._cache_serializable + def to_flyte_idl(self): """ :rtype: flyteidl.admin.task_pb2.TaskMetadata @@ -276,6 +289,7 @@ def to_flyte_idl(self): interruptible=self.interruptible, discovery_version=self.discovery_version, deprecated_error_message=self.deprecated_error_message, + cache_serializable=self.cache_serializable, ) if self.timeout: tm.timeout.FromTimedelta(self.timeout) @@ -295,6 +309,7 @@ def from_flyte_idl(cls, pb2_object): retries=_literals.RetryStrategy.from_flyte_idl(pb2_object.retries), discovery_version=pb2_object.discovery_version, deprecated_error_message=pb2_object.deprecated_error_message, + cache_serializable=pb2_object.cache_serializable, ) diff --git a/flytekit/sdk/tasks.py b/flytekit/sdk/tasks.py index 463a717d03..ed7607d584 100644 --- a/flytekit/sdk/tasks.py +++ b/flytekit/sdk/tasks.py @@ -138,6 +138,7 @@ def python_task( cache=False, timeout=None, environment=None, + cache_serialize=False, cls=None, ): """ @@ -225,6 +226,10 @@ def my_task(wf_params, int_list, sum_of_list): :param dict[Text,Text] environment: [optional] environment variables to set when executing this task. + :param bool cache_serialize: [optional] boolean describing if instances of this cachable task should be executed + in serial. This means only a single instances executes and other concurrent executions wait for it to complete + and reuse the cached outputs. + :param cls: This can be used to override the task implementation with a user-defined extension. The class provided must be a subclass of flytekit.common.tasks.sdk_runnable.SdkRunnableTask. A user can use this to inject bespoke logic into the base Flyte programming model. @@ -252,6 +257,7 @@ def wrapper(fn): discoverable=cache, timeout=timeout or _datetime.timedelta(seconds=0), environment=environment, + cache_serializable=cache_serialize, custom={}, ) @@ -281,6 +287,7 @@ def dynamic_task( allowed_failure_ratio=None, max_concurrency=None, environment=None, + cache_serialize=False, cls=None, ): """ @@ -378,6 +385,9 @@ def my_task(wf_params, out): This is a stand-in pending better concurrency controls for special use-cases. The existence of this parameter is not guaranteed between versions and therefore it is NOT recommended that it be used. :param dict[Text,Text] environment: [optional] environment variables to set when executing this task. + :param bool cache_serialize: [optional] boolean describing if instances of this cachable task should be executed + in serial. This means only a single instances executes and other concurrent executions wait for it to complete + and reuse the cached outputs. :param cls: This can be used to override the task implementation with a user-defined extension. The class provided must be a subclass of flytekit.common.tasks.sdk_runnable.SdkRunnableTask. Generally, it should be a subclass of flytekit.common.tasks.sdk_dynamic.SdkDynamicTask. A user can use this parameter to inject bespoke @@ -407,6 +417,7 @@ def wrapper(fn): allowed_failure_ratio=allowed_failure_ratio, max_concurrency=max_concurrency, environment=environment or {}, + cache_serializable=cache_serialize, custom={}, ) @@ -428,6 +439,7 @@ def spark_task( spark_conf=None, hadoop_conf=None, environment=None, + cache_serialize=False, cls=None, ): """ @@ -476,6 +488,9 @@ def sparky(wf_params, spark_context, a): :param dict[Text,Text] spark_conf: A definition of key-value pairs for spark config for the job. :param dict[Text,Text] hadoop_conf: A definition of key-value pairs for hadoop config for the job. :param dict[Text,Text] environment: [optional] environment variables to set when executing this task. + :param bool cache_serialize: [optional] boolean describing if instances of this cachable task should be executed + in serial. This means only a single instances executes and other concurrent executions wait for it to complete + and reuse the cached outputs. :param cls: This can be used to override the task implementation with a user-defined extension. The class provided must be a subclass of flytekit.common.tasks.sdk_runnable.SdkRunnableTask. Generally, it should be a subclass of flytekit.common.tasks.spark_task.SdkSparkTask. A user can use this parameter to inject bespoke @@ -498,6 +513,7 @@ def wrapper(fn): spark_conf=spark_conf or {}, hadoop_conf=hadoop_conf or {}, environment=environment or {}, + cache_serializable=cache_serialize, ) if _task_function: @@ -521,6 +537,7 @@ def generic_spark_task( spark_conf=None, hadoop_conf=None, environment=None, + cache_serialize=False, ): """ Create a generic spark task. This task will connect to a Spark cluster, configure the environment, @@ -544,6 +561,7 @@ def generic_spark_task( spark_conf=spark_conf or {}, hadoop_conf=hadoop_conf or {}, environment=environment or {}, + cache_serializable=cache_serialize, ) @@ -572,6 +590,7 @@ def hive_task( cache=False, timeout=None, environment=None, + cache_serialize=False, cls=None, ): """ @@ -657,6 +676,9 @@ def test_hive(wf_params, a): indefinitely. If a null timedelta is passed (i.e. timedelta(seconds=0)), the task will not timeout. :param dict[Text,Text] environment: Environment variables to set for the execution of the query-generating container. + :param bool cache_serialize: [optional] boolean describing if instances of this cachable task should be executed + in serial. This means only a single instances executes and other concurrent executions wait for it to complete + and reuse the cached outputs. :param cls: This can be used to override the task implementation with a user-defined extension. The class provided should be a subclass of flytekit.common.tasks.sdk_runnable.SdkRunnableTask. Generally, it should be a subclass of flytekit.common.tasks.hive_task.SdkHiveTask. A user can use this to inject bespoke logic into @@ -688,6 +710,7 @@ def wrapper(fn): cluster_label="", tags=[], environment=environment or {}, + cache_serializable=cache_serialize, ) if _task_function: @@ -716,6 +739,7 @@ def qubole_hive_task( cluster_label=None, tags=None, environment=None, + cache_serialize=False, cls=None, ): """ @@ -804,6 +828,9 @@ def test_hive(wf_params, a): passed to Qubole. :param dict[Text,Text] environment: Environment variables to set for the execution of the query-generating container. + :param bool cache_serialize: [optional] boolean describing if instances of this cachable task should be executed + in serial. This means only a single instances executes and other concurrent executions wait for it to complete + and reuse the cached outputs. :param cls: This can be used to override the task implementation with a user-defined extension. The class provided should be a subclass of flytekit.common.tasks.sdk_runnable.SdkRunnableTask. Generally, it should be a subclass of flytekit.common.tasks.hive_task.SdkHiveTask. A user can use this to inject bespoke logic into @@ -835,6 +862,7 @@ def wrapper(fn): cluster_label=cluster_label or "", tags=tags or [], environment=environment or {}, + cache_serializable=cache_serialize, ) # This is syntactic-sugar, so that when calling this decorator without args, you can either @@ -863,6 +891,7 @@ def sidecar_task( cache=False, timeout=None, environment=None, + cache_serialize=False, pod_spec=None, primary_container_name=None, annotations=None, @@ -990,6 +1019,10 @@ def a_sidecar_task(wfparams): :param dict[Text,Text] environment: [optional] environment variables to set when executing this task. + :param bool cache_serialize: [optional] boolean describing if instances of this cachable task should be executed + in serial. This means only a single instances executes and other concurrent executions wait for it to complete + and reuse the cached outputs. + :param k8s.io.api.core.v1.generated_pb2.PodSpec pod_spec: [optional] PodSpec to bring up alongside task execution. :param Text primary_container_name: primary container to monitor for the duration of the task. @@ -1027,6 +1060,7 @@ def wrapper(fn): discoverable=cache, timeout=timeout or _datetime.timedelta(seconds=0), environment=environment, + cache_serializable=cache_serialize, pod_spec=pod_spec, primary_container_name=primary_container_name, annotations=annotations, @@ -1059,6 +1093,7 @@ def dynamic_sidecar_task( allowed_failure_ratio=None, max_concurrency=None, environment=None, + cache_serialize=False, pod_spec=None, primary_container_name=None, annotations=None, @@ -1176,6 +1211,9 @@ def my_task(wf_params, out): This is a stand-in pending better concurrency controls for special use-cases. The existence of this parameter is not guaranteed between versions and therefore it is NOT recommended that it be used. :param dict[Text,Text] environment: [optional] environment variables to set when executing this task. + :param bool cache_serialize: [optional] boolean describing if instances of this cachable task should be executed + in serial. This means only a single instances executes and other concurrent executions wait for it to complete + and reuse the cached outputs. :param k8s.io.api.core.v1.generated_pb2.PodSpec pod_spec: PodSpec to bring up alongside task execution. :param Text primary_container_name: primary container to monitor for the duration of the task. :param dict[Text, Text] annotations: [optional] kubernetes annotations @@ -1209,6 +1247,7 @@ def wrapper(fn): allowed_failure_ratio=allowed_failure_ratio, max_concurrency=max_concurrency, environment=environment, + cache_serializable=cache_serialize, pod_spec=pod_spec, primary_container_name=primary_container_name, annotations=annotations, diff --git a/requirements-spark2.txt b/requirements-spark2.txt index 37a2664140..a0457279bc 100644 --- a/requirements-spark2.txt +++ b/requirements-spark2.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.8 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # make requirements-spark2.txt @@ -22,13 +22,13 @@ bcrypt==3.2.0 # via paramiko binaryornot==0.4.4 # via cookiecutter -black==21.10b0 +black==21.11b1 # via papermill bleach==4.1.0 # via nbconvert -boto3==1.20.2 +boto3==1.20.17 # via sagemaker-training -botocore==1.23.2 +botocore==1.23.17 # via # boto3 # s3transfer @@ -41,7 +41,7 @@ cffi==1.15.0 # pynacl chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.7 +charset-normalizer==2.0.8 # via requests checksumdir==1.2.0 # via flytekit @@ -58,7 +58,7 @@ cookiecutter==1.7.3 # via flytekit croniter==1.0.15 # via flytekit -cryptography==35.0.0 +cryptography==36.0.0 # via # paramiko # secretstorage @@ -72,24 +72,24 @@ defusedxml==0.7.1 # via nbconvert deprecated==1.2.13 # via flytekit -diskcache==5.2.1 +diskcache==5.3.0 # via flytekit docker-image-py==0.1.12 # via flytekit -docstring-parser==0.12 +docstring-parser==0.13 # via flytekit entrypoints==0.3 # via # jupyter-client # nbconvert # papermill -flyteidl==0.21.8 +flyteidl==0.21.11 # via flytekit gevent==21.8.0 # via sagemaker-training greenlet==1.1.2 # via gevent -grpcio==1.41.1 +grpcio==1.42.0 # via flytekit hmsclient==0.1.1 # via flytekit @@ -101,13 +101,13 @@ inotify_simple==1.2.1 # via sagemaker-training ipykernel==5.5.6 # via flytekit -ipython==7.29.0 +ipython==7.30.0 # via ipykernel ipython-genutils==0.2.0 # via # ipykernel # nbformat -jedi==0.18.0 +jedi==0.18.1 # via ipython jeepney==0.7.1 # via @@ -128,7 +128,7 @@ jsonschema==3.2.0 # via # -r requirements.in # nbformat -jupyter-client==7.0.6 +jupyter-client==7.1.0 # via # ipykernel # nbclient @@ -141,11 +141,11 @@ jupyterlab-pygments==0.1.2 # via nbconvert k8s-proto==0.0.3 # via flytekit -keyring==23.2.1 +keyring==23.4.0 # via flytekit markupsafe==2.0.1 # via jinja2 -marshmallow==3.14.0 +marshmallow==3.14.1 # via # dataclasses-json # marshmallow-enum @@ -164,18 +164,18 @@ mypy-extensions==0.4.3 # typing-inspect natsort==8.0.0 # via flytekit -nbclient==0.5.5 +nbclient==0.5.9 # via # nbconvert # papermill -nbconvert==6.2.0 +nbconvert==6.3.0 # via flytekit nbformat==5.1.3 # via # nbclient # nbconvert # papermill -nest-asyncio==1.5.1 +nest-asyncio==1.5.2 # via # jupyter-client # nbclient @@ -186,7 +186,7 @@ numpy==1.21.4 # pyarrow # sagemaker-training # scipy -packaging==21.2 +packaging==21.3 # via bleach pandas==1.3.4 # via flytekit @@ -194,9 +194,9 @@ pandocfilters==1.5.0 # via nbconvert papermill==2.3.3 # via flytekit -paramiko==2.8.0 +paramiko==2.8.1 # via sagemaker-training -parso==0.8.2 +parso==0.8.3 # via jedi pathspec==0.9.0 # via black @@ -208,7 +208,7 @@ platformdirs==2.4.0 # via black poyo==0.5.0 # via cookiecutter -prompt-toolkit==3.0.22 +prompt-toolkit==3.0.23 # via ipython protobuf==3.19.1 # via @@ -224,7 +224,7 @@ py==1.11.0 # via retry py4j==0.10.9.2 # via pyspark -pyarrow==6.0.0 +pyarrow==6.0.1 # via flytekit pycparser==2.21 # via cffi @@ -235,7 +235,7 @@ pygments==2.10.0 # nbconvert pynacl==1.4.0 # via paramiko -pyparsing==2.4.7 +pyparsing==3.0.6 # via packaging pyrsistent==0.18.0 # via jsonschema @@ -265,7 +265,7 @@ pyyaml==5.4.1 # papermill pyzmq==22.3.0 # via jupyter-client -regex==2021.11.9 +regex==2021.11.10 # via # black # docker-image-py @@ -275,7 +275,7 @@ requests==2.26.0 # flytekit # papermill # responses -responses==0.15.0 +responses==0.16.0 # via flytekit retry==0.9.2 # via flytekit @@ -285,7 +285,7 @@ s3transfer==0.5.0 # via boto3 sagemaker-training==3.9.2 # via flytekit -scipy==1.7.2 +scipy==1.7.3 # via sagemaker-training secretstorage==3.3.1 # via keyring @@ -335,7 +335,7 @@ traitlets==5.1.1 # nbclient # nbconvert # nbformat -typing-extensions==3.10.0.2 +typing-extensions==4.0.1 # via # black # typing-inspect diff --git a/requirements.txt b/requirements.txt index ee632aa7d5..344af69890 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.8 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # make requirements.txt @@ -20,13 +20,13 @@ bcrypt==3.2.0 # via paramiko binaryornot==0.4.4 # via cookiecutter -black==21.10b0 +black==21.11b1 # via papermill bleach==4.1.0 # via nbconvert -boto3==1.20.2 +boto3==1.20.17 # via sagemaker-training -botocore==1.23.2 +botocore==1.23.17 # via # boto3 # s3transfer @@ -39,7 +39,7 @@ cffi==1.15.0 # pynacl chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.7 +charset-normalizer==2.0.8 # via requests checksumdir==1.2.0 # via flytekit @@ -56,7 +56,7 @@ cookiecutter==1.7.3 # via flytekit croniter==1.0.15 # via flytekit -cryptography==35.0.0 +cryptography==36.0.0 # via # paramiko # secretstorage @@ -70,24 +70,24 @@ defusedxml==0.7.1 # via nbconvert deprecated==1.2.13 # via flytekit -diskcache==5.2.1 +diskcache==5.3.0 # via flytekit docker-image-py==0.1.12 # via flytekit -docstring-parser==0.12 +docstring-parser==0.13 # via flytekit entrypoints==0.3 # via # jupyter-client # nbconvert # papermill -flyteidl==0.21.8 +flyteidl==0.21.11 # via flytekit gevent==21.8.0 # via sagemaker-training greenlet==1.1.2 # via gevent -grpcio==1.41.1 +grpcio==1.42.0 # via flytekit hmsclient==0.1.1 # via flytekit @@ -99,13 +99,13 @@ inotify_simple==1.2.1 # via sagemaker-training ipykernel==5.5.6 # via flytekit -ipython==7.29.0 +ipython==7.30.0 # via ipykernel ipython-genutils==0.2.0 # via # ipykernel # nbformat -jedi==0.18.0 +jedi==0.18.1 # via ipython jeepney==0.7.1 # via @@ -126,7 +126,7 @@ jsonschema==3.2.0 # via # -r requirements.in # nbformat -jupyter-client==7.0.6 +jupyter-client==7.1.0 # via # ipykernel # nbclient @@ -139,11 +139,11 @@ jupyterlab-pygments==0.1.2 # via nbconvert k8s-proto==0.0.3 # via flytekit -keyring==23.2.1 +keyring==23.4.0 # via flytekit markupsafe==2.0.1 # via jinja2 -marshmallow==3.14.0 +marshmallow==3.14.1 # via # dataclasses-json # marshmallow-enum @@ -162,18 +162,18 @@ mypy-extensions==0.4.3 # typing-inspect natsort==8.0.0 # via flytekit -nbclient==0.5.5 +nbclient==0.5.9 # via # nbconvert # papermill -nbconvert==6.2.0 +nbconvert==6.3.0 # via flytekit nbformat==5.1.3 # via # nbclient # nbconvert # papermill -nest-asyncio==1.5.1 +nest-asyncio==1.5.2 # via # jupyter-client # nbclient @@ -184,7 +184,7 @@ numpy==1.21.4 # pyarrow # sagemaker-training # scipy -packaging==21.2 +packaging==21.3 # via bleach pandas==1.3.4 # via flytekit @@ -192,9 +192,9 @@ pandocfilters==1.5.0 # via nbconvert papermill==2.3.3 # via flytekit -paramiko==2.8.0 +paramiko==2.8.1 # via sagemaker-training -parso==0.8.2 +parso==0.8.3 # via jedi pathspec==0.9.0 # via black @@ -206,7 +206,7 @@ platformdirs==2.4.0 # via black poyo==0.5.0 # via cookiecutter -prompt-toolkit==3.0.22 +prompt-toolkit==3.0.23 # via ipython protobuf==3.19.1 # via @@ -222,7 +222,7 @@ py==1.11.0 # via retry py4j==0.10.9.2 # via pyspark -pyarrow==6.0.0 +pyarrow==6.0.1 # via flytekit pycparser==2.21 # via cffi @@ -233,7 +233,7 @@ pygments==2.10.0 # nbconvert pynacl==1.4.0 # via paramiko -pyparsing==2.4.7 +pyparsing==3.0.6 # via packaging pyrsistent==0.18.0 # via jsonschema @@ -263,7 +263,7 @@ pyyaml==5.4.1 # papermill pyzmq==22.3.0 # via jupyter-client -regex==2021.11.9 +regex==2021.11.10 # via # black # docker-image-py @@ -273,7 +273,7 @@ requests==2.26.0 # flytekit # papermill # responses -responses==0.15.0 +responses==0.16.0 # via flytekit retry==0.9.2 # via flytekit @@ -283,7 +283,7 @@ s3transfer==0.5.0 # via boto3 sagemaker-training==3.9.2 # via flytekit -scipy==1.7.2 +scipy==1.7.3 # via sagemaker-training secretstorage==3.3.1 # via keyring @@ -333,7 +333,7 @@ traitlets==5.1.1 # nbclient # nbconvert # nbformat -typing-extensions==3.10.0.2 +typing-extensions==4.0.1 # via # black # typing-inspect diff --git a/tests/flytekit/common/parameterizers.py b/tests/flytekit/common/parameterizers.py index 92031cf957..10a090d0c6 100644 --- a/tests/flytekit/common/parameterizers.py +++ b/tests/flytekit/common/parameterizers.py @@ -116,8 +116,9 @@ interruptible, discovery_version, deprecated, + cache_serializable, ) - for discoverable, runtime_metadata, timeout, retry_strategy, interruptible, discovery_version, deprecated in product( + for discoverable, runtime_metadata, timeout, retry_strategy, interruptible, discovery_version, deprecated, cache_serializable in product( [True, False], LIST_OF_RUNTIME_METADATA, [timedelta(days=i) for i in range(3)], @@ -125,6 +126,7 @@ LIST_OF_INTERRUPTIBLE, ["1.0"], ["deprecated"], + [True, False], ) ] diff --git a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt index a37b0c3af8..f9ecc1e38d 100644 --- a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt +++ b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.8 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # make tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt @@ -14,7 +14,7 @@ cffi==1.15.0 # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.7 +charset-normalizer==2.0.8 # via requests checksumdir==1.2.0 # via flytekit @@ -28,7 +28,7 @@ cookiecutter==1.7.3 # via flytekit croniter==1.0.15 # via flytekit -cryptography==35.0.0 +cryptography==36.0.0 # via secretstorage cycler==0.11.0 # via matplotlib @@ -38,17 +38,19 @@ decorator==5.1.0 # via retry deprecated==1.2.13 # via flytekit -diskcache==5.2.1 +diskcache==5.3.0 # via flytekit docker-image-py==0.1.12 # via flytekit -docstring-parser==0.12 +docstring-parser==0.13 # via flytekit -flyteidl==0.21.8 +flyteidl==0.21.11 # via flytekit flytekit==0.24.0 # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in -grpcio==1.41.1 +fonttools==4.28.2 + # via matplotlib +grpcio==1.42.0 # via flytekit idna==3.3 # via requests @@ -66,13 +68,13 @@ jinja2-time==0.2.0 # via cookiecutter joblib==1.1.0 # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in -keyring==23.2.1 +keyring==23.4.0 # via flytekit kiwisolver==1.3.2 # via matplotlib markupsafe==2.0.1 # via jinja2 -marshmallow==3.14.0 +marshmallow==3.14.1 # via # dataclasses-json # marshmallow-enum @@ -81,7 +83,7 @@ marshmallow-enum==1.5.1 # via dataclasses-json marshmallow-jsonschema==0.13.0 # via flytekit -matplotlib==3.4.3 +matplotlib==3.5.0 # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in mypy-extensions==0.4.3 # via typing-inspect @@ -93,8 +95,12 @@ numpy==1.21.4 # opencv-python # pandas # pyarrow -opencv-python==4.5.4.58 +opencv-python==4.5.4.60 # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in +packaging==21.3 + # via + # matplotlib + # setuptools-scm pandas==1.3.4 # via flytekit pillow==8.4.0 @@ -107,12 +113,14 @@ protobuf==3.19.1 # flytekit py==1.11.0 # via retry -pyarrow==6.0.0 +pyarrow==6.0.1 # via flytekit pycparser==2.21 # via cffi -pyparsing==3.0.5 - # via matplotlib +pyparsing==3.0.6 + # via + # matplotlib + # packaging python-dateutil==2.8.1 # via # arrow @@ -130,19 +138,21 @@ pytz==2018.4 # via # flytekit # pandas -regex==2021.11.9 +regex==2021.11.10 # via docker-image-py requests==2.26.0 # via # cookiecutter # flytekit # responses -responses==0.15.0 +responses==0.16.0 # via flytekit retry==0.9.2 # via flytekit secretstorage==3.3.1 # via keyring +setuptools-scm==6.3.2 + # via matplotlib six==1.16.0 # via # cookiecutter @@ -156,7 +166,9 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==3.10.0.2 +tomli==1.2.2 + # via setuptools-scm +typing-extensions==4.0.1 # via typing-inspect typing-inspect==0.7.1 # via dataclasses-json @@ -175,3 +187,6 @@ wrapt==1.13.3 # flytekit zipp==3.6.0 # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/tests/flytekit/unit/common_tests/tasks/test_sdk_runnable.py b/tests/flytekit/unit/common_tests/tasks/test_sdk_runnable.py index 29402bea55..9ac2d8b119 100644 --- a/tests/flytekit/unit/common_tests/tasks/test_sdk_runnable.py +++ b/tests/flytekit/unit/common_tests/tasks/test_sdk_runnable.py @@ -30,6 +30,7 @@ def add_one(wf_params, value_in, value_out): False, None, {}, + False, None, ) t.add_inputs({"value_in": interface.Variable(primitives.Integer.to_flyte_literal_type(), "")}) diff --git a/tests/flytekit/unit/common_tests/test_workflow_promote.py b/tests/flytekit/unit/common_tests/test_workflow_promote.py index cdd1231107..f165f4c231 100644 --- a/tests/flytekit/unit/common_tests/test_workflow_promote.py +++ b/tests/flytekit/unit/common_tests/test_workflow_promote.py @@ -60,6 +60,7 @@ def get_sample_task_metadata(): True, "0.1.1b0", "This is deprecated!", + True, ) diff --git a/tests/flytekit/unit/core/test_python_function_task.py b/tests/flytekit/unit/core/test_python_function_task.py index 7101b585d3..708bacfe50 100644 --- a/tests/flytekit/unit/core/test_python_function_task.py +++ b/tests/flytekit/unit/core/test_python_function_task.py @@ -89,10 +89,36 @@ def foo(i: int): def test_metadata(): - @task(cache=True, cache_version="1.0") + # test cache, cache_serialize, and cache_version are correctly set + @task(cache=True, cache_serialize=True, cache_version="1.0") def foo(i: str): print(f"{i}") - metadata = foo.metadata - assert metadata.cache is True - assert metadata.cache_version == "1.0" + foo_metadata = foo.metadata + assert foo_metadata.cache is True + assert foo_metadata.cache_serialize is True + assert foo_metadata.cache_version == "1.0" + + # test cache, cache_serialize, and cache_version at no unecessarily set + @task() + def bar(i: str): + print(f"{i}") + + bar_metadata = bar.metadata + assert bar_metadata.cache is False + assert bar_metadata.cache_serialize is False + assert bar_metadata.cache_version == "" + + # test missing cache_version + with pytest.raises(ValueError): + + @task(cache=True) + def foo_missing_cache_version(i: str): + print(f"{i}") + + # test missing cache + with pytest.raises(ValueError): + + @task(cache_serialize=True) + def foo_missing_cache(i: str): + print(f"{i}") diff --git a/tests/flytekit/unit/models/test_tasks.py b/tests/flytekit/unit/models/test_tasks.py index e849b0e813..d479ded286 100644 --- a/tests/flytekit/unit/models/test_tasks.py +++ b/tests/flytekit/unit/models/test_tasks.py @@ -70,6 +70,7 @@ def test_task_metadata(): True, "0.1.1b0", "This is deprecated!", + True, ) assert obj.discoverable is True @@ -136,6 +137,7 @@ def test_task_template__k8s_pod_target(): False, "1.0", "deprecated", + False, ), interface_models.TypedInterface( # inputs diff --git a/tests/flytekit/unit/models/test_workflow_closure.py b/tests/flytekit/unit/models/test_workflow_closure.py index 3e19a80657..3a42f5af81 100644 --- a/tests/flytekit/unit/models/test_workflow_closure.py +++ b/tests/flytekit/unit/models/test_workflow_closure.py @@ -35,6 +35,7 @@ def test_workflow_closure(): True, "0.1.1b0", "This is deprecated!", + True, ) cpu_resource = _task.Resources.ResourceEntry(_task.Resources.ResourceName.CPU, "1")