diff --git a/Makefile b/Makefile index b5464e205e..404c1eb1fd 100644 --- a/Makefile +++ b/Makefile @@ -50,7 +50,8 @@ test: lint unit_test .PHONY: unit_test unit_test: - FLYTE_SDK_USE_STRUCTURED_DATASET=TRUE pytest tests/flytekit/unit tests/flytekit_compatibility + FLYTE_SDK_USE_STRUCTURED_DATASET=FALSE pytest tests/flytekit_compatibility && \ + FLYTE_SDK_USE_STRUCTURED_DATASET=TRUE pytest tests/flytekit/unit requirements-spark2.txt: export CUSTOM_COMPILE_COMMAND := make requirements-spark2.txt requirements-spark2.txt: requirements-spark2.in install-piptools diff --git a/dev-requirements.txt b/dev-requirements.txt index 0a307eb23f..f8bd6af672 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.8 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # make dev-requirements.txt @@ -32,6 +32,7 @@ certifi==2021.10.8 # requests cffi==1.15.0 # via + # -c requirements.txt # bcrypt # cryptography # pynacl @@ -71,7 +72,10 @@ croniter==1.3.4 # -c requirements.txt # flytekit cryptography==36.0.1 - # via paramiko + # via + # -c requirements.txt + # paramiko + # secretstorage dataclasses-json==0.5.6 # via # -c requirements.txt @@ -112,7 +116,7 @@ docstring-parser==0.13 # flytekit filelock==3.6.0 # via virtualenv -flyteidl==0.22.3 +flyteidl==0.23.0 # via # -c requirements.txt # flytekit @@ -133,9 +137,9 @@ google-cloud-core==2.2.2 # via google-cloud-bigquery google-crc32c==1.3.0 # via google-resumable-media -google-resumable-media==2.2.1 +google-resumable-media==2.3.0 # via google-cloud-bigquery -googleapis-common-protos==1.54.0 +googleapis-common-protos==1.55.0 # via # -c requirements.txt # flyteidl @@ -156,12 +160,17 @@ idna==3.3 # via # -c requirements.txt # requests -importlib-metadata==4.11.1 +importlib-metadata==4.11.2 # via # -c requirements.txt # keyring iniconfig==1.1.1 # via pytest +jeepney==0.7.1 + # via + # -c requirements.txt + # keyring + # secretstorage jinja2==3.0.3 # via # -c requirements.txt @@ -275,7 +284,9 @@ pyasn1==0.4.8 pyasn1-modules==0.2.8 # via google-auth pycparser==2.21 - # via cffi + # via + # -c requirements.txt + # cffi pynacl==1.5.0 # via paramiko pyparsing==3.0.7 @@ -307,7 +318,7 @@ python-json-logger==2.0.2 # via # -c requirements.txt # flytekit -python-slugify==6.1.0 +python-slugify==6.1.1 # via # -c requirements.txt # cookiecutter @@ -349,6 +360,10 @@ retry==0.9.2 # flytekit rsa==4.8 # via google-auth +secretstorage==3.3.1 + # via + # -c requirements.txt + # keyring six==1.16.0 # via # -c requirements.txt @@ -399,7 +414,7 @@ urllib3==1.26.8 # flytekit # requests # responses -virtualenv==20.13.1 +virtualenv==20.13.2 # via pre-commit websocket-client==0.59.0 # via diff --git a/doc-requirements.txt b/doc-requirements.txt index e6455ed494..07ef3d0d5b 100644 --- a/doc-requirements.txt +++ b/doc-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.8 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # make doc-requirements.txt @@ -10,7 +10,7 @@ alabaster==0.7.12 # via sphinx arrow==1.2.2 # via jinja2-time -astroid==2.9.3 +astroid==2.10.0 # via sphinx-autoapi babel==2.9.1 # via sphinx @@ -42,7 +42,9 @@ cookiecutter==1.7.3 croniter==1.3.4 # via flytekit cryptography==36.0.1 - # via -r doc-requirements.in + # via + # -r doc-requirements.in + # secretstorage css-html-js-minify==2.5.5 # via sphinx-material dataclasses-json==0.5.6 @@ -61,7 +63,7 @@ docutils==0.17.1 # via # sphinx # sphinx-panels -flyteidl==0.22.3 +flyteidl==0.23.0 # via flytekit furo @ git+https://github.com/flyteorg/furo@main # via -r doc-requirements.in @@ -75,10 +77,14 @@ idna==3.3 # via requests imagesize==1.3.0 # via sphinx -importlib-metadata==4.11.1 +importlib-metadata==4.11.2 # via # keyring # sphinx +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -125,10 +131,7 @@ protobuf==3.19.4 # googleapis-common-protos # protoc-gen-swagger protoc-gen-swagger==0.1.0 - # via - # flyteidl - # flytekit - + # via flyteidl py==1.11.0 # via retry pyarrow==6.0.1 @@ -149,7 +152,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify[unidecode]==6.1.0 +python-slugify[unidecode]==6.1.1 # via # cookiecutter # sphinx-material @@ -174,6 +177,8 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter diff --git a/flytekit/__init__.py b/flytekit/__init__.py index 6e2e614702..8089e8bd74 100644 --- a/flytekit/__init__.py +++ b/flytekit/__init__.py @@ -169,6 +169,7 @@ from flytekit.core.context_manager import ExecutionParameters, FlyteContext, FlyteContextManager from flytekit.core.data_persistence import DataPersistence, DataPersistencePlugins from flytekit.core.dynamic_workflow_task import dynamic +from flytekit.core.hash import HashMethod from flytekit.core.launch_plan import LaunchPlan from flytekit.core.map_task import map_task from flytekit.core.notification import Email, PagerDuty, Slack diff --git a/flytekit/core/hash.py b/flytekit/core/hash.py index 8b2559f696..2aca94e846 100644 --- a/flytekit/core/hash.py +++ b/flytekit/core/hash.py @@ -1,3 +1,23 @@ +from typing import Callable, Generic, TypeVar + +T = TypeVar("T") + + class HashOnReferenceMixin(object): def __hash__(self): return hash(id(self)) + + +class HashMethod(Generic[T]): + """ + Flyte-specific object used to wrap the hash function for a specific type + """ + + def __init__(self, function: Callable[[T], str]): + self._function = function + + def calculate(self, obj: T) -> str: + """ + Calculate hash for `obj`. + """ + return self._function(obj) diff --git a/flytekit/core/local_cache.py b/flytekit/core/local_cache.py index 9180224207..4afab73955 100644 --- a/flytekit/core/local_cache.py +++ b/flytekit/core/local_cache.py @@ -1,16 +1,42 @@ +import base64 from typing import Optional +import cloudpickle from diskcache import Cache -from flytekit.models.literals import LiteralMap +from flytekit.models.literals import Literal, LiteralCollection, LiteralMap # Location on the filesystem where serialized objects will be stored # TODO: read from config CACHE_LOCATION = "~/.flyte/local-cache" +def _recursive_hash_placement(literal: Literal) -> Literal: + if literal.collection is not None: + literals = [_recursive_hash_placement(literal) for literal in literal.collection.literals] + return Literal(collection=LiteralCollection(literals=literals)) + elif literal.map is not None: + literal_map = {} + for key, literal in literal.map.literals.items(): + literal_map[key] = _recursive_hash_placement(literal) + return Literal(map=LiteralMap(literal_map)) + + # Base case + if literal.hash is not None: + return Literal(hash=literal.hash) + else: + return literal + + def _calculate_cache_key(task_name: str, cache_version: str, input_literal_map: LiteralMap) -> str: - return f"{task_name}-{cache_version}-{input_literal_map}" + # Traverse the literals and replace the literal with a new literal that only contains the hash + literal_map_overridden = {} + for key, literal in input_literal_map.literals.items(): + literal_map_overridden[key] = _recursive_hash_placement(literal) + + # Pickle the literal map and use base64 encoding to generate a representation of it + b64_encoded = base64.b64encode(cloudpickle.dumps(LiteralMap(literal_map_overridden))) + return f"{task_name}-{cache_version}-{b64_encoded}" class LocalTaskCache(object): diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index 62f2daadc0..1a84c744a8 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -10,11 +10,6 @@ from abc import ABC, abstractmethod from typing import NamedTuple, Optional, Type, cast -try: - from typing import Annotated, get_args, get_origin -except ImportError: - from typing_extensions import Annotated, get_origin, get_args - from dataclasses_json import DataClassJsonMixin, dataclass_json from google.protobuf import json_format as _json_format from google.protobuf import reflection as _proto_reflection @@ -24,9 +19,11 @@ from google.protobuf.struct_pb2 import Struct from marshmallow_enum import EnumField, LoadDumpOptions from marshmallow_jsonschema import JSONSchema +from typing_extensions import Annotated, get_args, get_origin from flytekit.core.annotation import FlyteAnnotation from flytekit.core.context_manager import FlyteContext +from flytekit.core.hash import HashMethod from flytekit.core.type_helpers import load_type_from_tag from flytekit.exceptions import user as user_exceptions from flytekit.loggers import logger @@ -56,10 +53,13 @@ class TypeTransformer(typing.Generic[T]): Base transformer type that should be implemented for every python native type that can be handled by flytekit """ - def __init__(self, name: str, t: Type[T], enable_type_assertions: bool = True): + def __init__(self, name: str, t: Type[T], enable_type_assertions: bool = True, hash_overridable: bool = False): self._t = t self._name = name self._type_assertions_enabled = enable_type_assertions + # `hash_overridable` indicates that the literals produced by this type transformer can set their hashes if needed. + # See (link to documentation where this feature is explained). + self._hash_overridable = hash_overridable @property def name(self): @@ -79,6 +79,10 @@ def type_assertions_enabled(self) -> bool: """ return self._type_assertions_enabled + @property + def hash_overridable(self) -> bool: + return self._hash_overridable + def assert_type(self, t: Type[T], v: T): if not hasattr(t, "__origin__") and not isinstance(v, t): raise TypeError(f"Type of Val '{v}' is not an instance of {t}") @@ -640,7 +644,25 @@ def to_literal(cls, ctx: FlyteContext, python_val: typing.Any, python_type: Type transformer = cls.get_transformer(python_type) if transformer.type_assertions_enabled: transformer.assert_type(python_type, python_val) + + # In case the value is an annotated type we inspect the annotations and look for hash-related annotations. + hash = None + if transformer.hash_overridable and get_origin(python_type) is Annotated: + # We are now dealing with one of two cases: + # 1. The annotated type is a `HashMethod`, which indicates that we should we should produce the hash using + # the method indicated in the annotation. + # 2. The annotated type is being used for a different purpose other than calculating hash values, in which case + # we should just continue. + for annotation in get_args(python_type)[1:]: + if not isinstance(annotation, HashMethod): + continue + hash = annotation.calculate(python_val) + break + lv = transformer.to_literal(ctx, python_val, python_type, expected) + + if hash is not None: + lv.hash = hash return lv @classmethod @@ -852,6 +874,7 @@ def to_literal( for k, v in python_val.items(): if type(k) != str: raise ValueError("Flyte MapType expects all keys to be strings") + # TODO: log a warning for Annotated objects that contain HashMethod k_type, v_type = self.get_dict_types(python_type) lit_map[k] = TypeEngine.to_literal(ctx, v, v_type, expected.map_value_type) return Literal(map=LiteralMap(literals=lit_map)) diff --git a/flytekit/models/literals.py b/flytekit/models/literals.py index bc398ab7a7..70470fda6b 100644 --- a/flytekit/models/literals.py +++ b/flytekit/models/literals.py @@ -796,7 +796,9 @@ def from_flyte_idl(cls, pb2_object): class Literal(_common.FlyteIdlEntity): - def __init__(self, scalar: Scalar = None, collection: LiteralCollection = None, map: LiteralMap = None): + def __init__( + self, scalar: Scalar = None, collection: LiteralCollection = None, map: LiteralMap = None, hash: str = None + ): """ This IDL message represents a literal value in the Flyte ecosystem. @@ -807,6 +809,7 @@ def __init__(self, scalar: Scalar = None, collection: LiteralCollection = None, self._scalar = scalar self._collection = collection self._map = map + self._hash = hash @property def scalar(self): @@ -840,6 +843,18 @@ def value(self): """ return self.scalar or self.collection or self.map + @property + def hash(self): + """ + If not None, this value holds a hash that represents the literal for caching purposes. + :rtype: str + """ + return self._hash + + @hash.setter + def hash(self, value): + self._hash = value + def to_flyte_idl(self): """ :rtype: flyteidl.core.literals_pb2.Literal @@ -848,6 +863,7 @@ def to_flyte_idl(self): scalar=self.scalar.to_flyte_idl() if self.scalar is not None else None, collection=self.collection.to_flyte_idl() if self.collection is not None else None, map=self.map.to_flyte_idl() if self.map is not None else None, + hash=self.hash, ) @classmethod @@ -864,4 +880,5 @@ def from_flyte_idl(cls, pb2_object): scalar=Scalar.from_flyte_idl(pb2_object.scalar) if pb2_object.HasField("scalar") else None, collection=collection, map=LiteralMap.from_flyte_idl(pb2_object.map) if pb2_object.HasField("map") else None, + hash=pb2_object.hash if pb2_object.hash else None, ) diff --git a/flytekit/types/schema/types_pandas.py b/flytekit/types/schema/types_pandas.py index 0edf024b08..fa0245236c 100644 --- a/flytekit/types/schema/types_pandas.py +++ b/flytekit/types/schema/types_pandas.py @@ -112,6 +112,9 @@ class PandasDataFrameTransformer(TypeTransformer[pandas.DataFrame]): def __init__(self): super().__init__("PandasDataFrame<->GenericSchema", pandas.DataFrame) self._parquet_engine = _PARQUETIO_ENGINES[sdk.PARQUET_ENGINE.get()] + # Pandas dataframes can have their hashes overriden to facilitate the case of caching pandas dataframes by + # value. + self._hash_overridable = True @staticmethod def _get_schema_type() -> SchemaType: diff --git a/flytekit/types/structured/structured_dataset.py b/flytekit/types/structured/structured_dataset.py index 819bc012cc..7001846a3d 100644 --- a/flytekit/types/structured/structured_dataset.py +++ b/flytekit/types/structured/structured_dataset.py @@ -379,6 +379,9 @@ def __init__(self): super().__init__("StructuredDataset Transformer", StructuredDataset) self._type_assertions_enabled = False + # Instances of StructuredDataset opt-in to the ability of being cached. + self._hash_overridable = True + @classmethod def register(cls, h: Handlers, default_for_type: Optional[bool] = True, override: Optional[bool] = False): """ diff --git a/plugins/flytekit-aws-athena/requirements.txt b/plugins/flytekit-aws-athena/requirements.txt index a1acae494f..bb210d8c13 100644 --- a/plugins/flytekit-aws-athena/requirements.txt +++ b/plugins/flytekit-aws-athena/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in @@ -12,13 +12,15 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.4 # via # cookiecutter # flytekit @@ -26,7 +28,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.4 # via flytekit cryptography==36.0.1 # via secretstorage @@ -42,16 +44,22 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 +flyteidl==0.23.0 # via flytekit -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-athena -grpcio==1.43.0 +googleapis-common-protos==1.55.0 + # via flyteidl +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.2 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -60,7 +68,7 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -markupsafe==2.0.1 +markupsafe==2.1.0 # via jinja2 marshmallow==3.14.1 # via @@ -79,7 +87,7 @@ numpy==1.22.2 # via # pandas # pyarrow -pandas==1.4.0 +pandas==1.4.1 # via flytekit poyo==0.5.0 # via cookiecutter @@ -87,6 +95,10 @@ protobuf==3.19.4 # via # flyteidl # flytekit + # googleapis-common-protos + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl py==1.11.0 # via retry pyarrow==6.0.1 @@ -101,7 +113,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -109,7 +121,7 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.1.18 +regex==2022.3.2 # via docker-image-py requests==2.27.1 # via @@ -120,6 +132,8 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -131,7 +145,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-aws-batch/requirements.txt b/plugins/flytekit-aws-batch/requirements.txt index 9181490234..80a3fcf51f 100644 --- a/plugins/flytekit-aws-batch/requirements.txt +++ b/plugins/flytekit-aws-batch/requirements.txt @@ -1,24 +1,26 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in # -e file:.#egg=flytekitplugins-awsbatch # via -r requirements.in -arrow==1.2.1 +arrow==1.2.2 # via jinja2-time binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.10 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.4 # via # cookiecutter # flytekit @@ -26,8 +28,10 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.4 # via flytekit +cryptography==36.0.1 + # via secretstorage dataclasses-json==0.5.6 # via flytekit decorator==5.1.1 @@ -40,16 +44,22 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.21.23 +flyteidl==0.23.0 # via flytekit -flytekit==0.26.0 +flytekit==0.30.3 # via flytekitplugins-awsbatch -grpcio==1.43.0 +googleapis-common-protos==1.55.0 + # via flyteidl +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.2 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -58,7 +68,7 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -markupsafe==2.0.1 +markupsafe==2.1.0 # via jinja2 marshmallow==3.14.1 # via @@ -71,25 +81,31 @@ marshmallow-jsonschema==0.13.0 # via flytekit mypy-extensions==0.4.3 # via typing-inspect -natsort==8.0.2 +natsort==8.1.0 # via flytekit -numpy==1.22.1 +numpy==1.22.2 # via # pandas # pyarrow -pandas==1.3.5 +pandas==1.4.1 # via flytekit poyo==0.5.0 # via cookiecutter -protobuf==3.19.3 +protobuf==3.19.4 # via # flyteidl # flytekit + # googleapis-common-protos + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl py==1.11.0 # via retry pyarrow==6.0.1 # via flytekit -python-dateutil==2.8.1 +pycparser==2.21 + # via cffi +python-dateutil==2.8.2 # via # arrow # croniter @@ -97,7 +113,7 @@ python-dateutil==2.8.1 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -105,32 +121,34 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.1.18 +regex==2022.3.2 # via docker-image-py requests==2.27.1 # via # cookiecutter # flytekit # responses -responses==0.17.0 +responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter - # flytekit # grpcio # python-dateutil - # responses sortedcontainers==2.4.0 # via flytekit statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 - # via typing-inspect +typing-extensions==4.1.1 + # via + # flytekit + # typing-inspect typing-inspect==0.7.1 # via dataclasses-json urllib3==1.26.8 diff --git a/plugins/flytekit-aws-sagemaker/requirements.txt b/plugins/flytekit-aws-sagemaker/requirements.txt index 11396f213e..f4b02da229 100644 --- a/plugins/flytekit-aws-sagemaker/requirements.txt +++ b/plugins/flytekit-aws-sagemaker/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in @@ -12,9 +12,9 @@ bcrypt==3.2.0 # via paramiko binaryornot==0.4.4 # via cookiecutter -boto3==1.20.50 +boto3==1.21.10 # via sagemaker-training -botocore==1.23.50 +botocore==1.24.10 # via # boto3 # s3transfer @@ -27,11 +27,11 @@ cffi==1.15.0 # pynacl chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.4 # via # cookiecutter # flytekit @@ -39,7 +39,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.4 # via flytekit cryptography==36.0.1 # via @@ -57,22 +57,28 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 +flyteidl==0.23.0 # via flytekit -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-awssagemaker gevent==21.12.0 # via sagemaker-training +googleapis-common-protos==1.55.0 + # via flyteidl greenlet==1.1.2 # via gevent -grpcio==1.43.0 +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.2 # via keyring -inotify-simple==1.2.1 +inotify_simple==1.2.1 # via sagemaker-training +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -85,7 +91,7 @@ jmespath==0.10.0 # botocore keyring==23.5.0 # via flytekit -markupsafe==2.0.1 +markupsafe==2.1.0 # via jinja2 marshmallow==3.14.1 # via @@ -106,7 +112,7 @@ numpy==1.22.2 # pyarrow # sagemaker-training # scipy -pandas==1.4.0 +pandas==1.4.1 # via flytekit paramiko==2.9.2 # via sagemaker-training @@ -116,7 +122,11 @@ protobuf==3.19.4 # via # flyteidl # flytekit + # googleapis-common-protos + # protoc-gen-swagger # sagemaker-training +protoc-gen-swagger==0.1.0 + # via flyteidl psutil==5.9.0 # via sagemaker-training py==1.11.0 @@ -136,7 +146,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -144,7 +154,7 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.1.18 +regex==2022.3.2 # via docker-image-py requests==2.27.1 # via @@ -157,12 +167,14 @@ retry==0.9.2 # via flytekit retrying==1.3.3 # via sagemaker-training -s3transfer==0.5.1 +s3transfer==0.5.2 # via boto3 sagemaker-training==3.9.2 # via flytekitplugins-awssagemaker scipy==1.8.0 # via sagemaker-training +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # bcrypt @@ -177,7 +189,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect @@ -199,9 +211,9 @@ wrapt==1.13.3 # flytekit zipp==3.7.0 # via importlib-metadata -zope-event==4.5.0 +zope.event==4.5.0 # via gevent -zope-interface==5.4.0 +zope.interface==5.4.0 # via gevent # The following packages are considered to be unsafe in a requirements file: diff --git a/plugins/flytekit-bigquery/requirements.txt b/plugins/flytekit-bigquery/requirements.txt index f0bc647453..aef0a38759 100644 --- a/plugins/flytekit-bigquery/requirements.txt +++ b/plugins/flytekit-bigquery/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in @@ -18,11 +18,11 @@ cffi==1.15.0 # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.4 # via # cookiecutter # flytekit @@ -30,7 +30,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.4 # via flytekit cryptography==36.0.1 # via secretstorage @@ -46,9 +46,9 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 +flyteidl==0.23.0 # via flytekit -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-bigquery google-api-core[grpc]==2.5.0 # via @@ -58,29 +58,30 @@ google-auth==2.6.0 # via # google-api-core # google-cloud-core -google-cloud-bigquery==2.32.0 +google-cloud-bigquery==2.34.0 # via flytekitplugins-bigquery google-cloud-core==2.2.2 # via google-cloud-bigquery google-crc32c==1.3.0 # via google-resumable-media -google-resumable-media==2.2.0 +google-resumable-media==2.3.0 # via google-cloud-bigquery -googleapis-common-protos==1.54.0 +googleapis-common-protos==1.55.0 # via + # flyteidl # google-api-core # grpcio-status -grpcio==1.43.0 +grpcio==1.44.0 # via # flytekit # google-api-core # google-cloud-bigquery # grpcio-status -grpcio-status==1.43.0 +grpcio-status==1.44.0 # via google-api-core idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.2 # via keyring jeepney==0.7.1 # via @@ -94,7 +95,7 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -markupsafe==2.0.1 +markupsafe==2.1.0 # via jinja2 marshmallow==3.14.1 # via @@ -115,11 +116,11 @@ numpy==1.22.2 # pyarrow packaging==21.3 # via google-cloud-bigquery -pandas==1.4.0 +pandas==1.4.1 # via flytekit poyo==0.5.0 # via cookiecutter -proto-plus==1.20.0 +proto-plus==1.20.3 # via google-cloud-bigquery protobuf==3.19.4 # via @@ -130,6 +131,9 @@ protobuf==3.19.4 # googleapis-common-protos # grpcio-status # proto-plus + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl py==1.11.0 # via retry pyarrow==6.0.1 @@ -153,7 +157,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -161,7 +165,7 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.1.18 +regex==2022.3.2 # via docker-image-py requests==2.27.1 # via @@ -190,7 +194,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-data-fsspec/requirements.txt b/plugins/flytekit-data-fsspec/requirements.txt index 4dc89fa500..b3f506d572 100644 --- a/plugins/flytekit-data-fsspec/requirements.txt +++ b/plugins/flytekit-data-fsspec/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in @@ -10,17 +10,19 @@ arrow==1.2.2 # via jinja2-time binaryornot==0.4.4 # via cookiecutter -botocore==1.23.24 +botocore==1.24.10 # via flytekitplugins-data-fsspec certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.4 # via # cookiecutter # flytekit @@ -28,8 +30,10 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.4 # via flytekit +cryptography==36.0.1 + # via secretstorage dataclasses-json==0.5.6 # via flytekit decorator==5.1.1 @@ -42,18 +46,24 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 +flyteidl==0.23.0 # via flytekit -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-data-fsspec -fsspec==2022.1.0 +fsspec==2022.2.0 # via flytekitplugins-data-fsspec -grpcio==1.43.0 +googleapis-common-protos==1.55.0 + # via flyteidl +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.2 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -64,7 +74,7 @@ jmespath==0.10.0 # via botocore keyring==23.5.0 # via flytekit -markupsafe==2.0.1 +markupsafe==2.1.0 # via jinja2 marshmallow==3.14.1 # via @@ -83,7 +93,7 @@ numpy==1.22.2 # via # pandas # pyarrow -pandas==1.4.0 +pandas==1.4.1 # via flytekit poyo==0.5.0 # via cookiecutter @@ -91,10 +101,16 @@ protobuf==3.19.4 # via # flyteidl # flytekit + # googleapis-common-protos + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl py==1.11.0 # via retry pyarrow==6.0.1 # via flytekit +pycparser==2.21 + # via cffi python-dateutil==2.8.2 # via # arrow @@ -104,7 +120,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -112,7 +128,7 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.1.18 +regex==2022.3.2 # via docker-image-py requests==2.27.1 # via @@ -123,6 +139,8 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -134,7 +152,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-dolt/requirements.txt b/plugins/flytekit-dolt/requirements.txt index 01dfe2cb6b..fc9dc9f8b7 100644 --- a/plugins/flytekit-dolt/requirements.txt +++ b/plugins/flytekit-dolt/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in @@ -12,13 +12,15 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.4 # via # cookiecutter # flytekit @@ -26,7 +28,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.4 # via flytekit cryptography==36.0.1 # via secretstorage @@ -48,16 +50,22 @@ dolt-integrations==0.1.5 # via flytekitplugins-dolt doltcli==0.1.17 # via dolt-integrations -flyteidl==0.22.0 +flyteidl==0.23.0 # via flytekit -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-dolt -grpcio==1.43.0 +googleapis-common-protos==1.55.0 + # via flyteidl +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.2 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -66,7 +74,7 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -markupsafe==2.0.1 +markupsafe==2.1.0 # via jinja2 marshmallow==3.14.1 # via @@ -85,7 +93,7 @@ numpy==1.22.2 # via # pandas # pyarrow -pandas==1.4.0 +pandas==1.4.1 # via # dolt-integrations # flytekit @@ -95,6 +103,10 @@ protobuf==3.19.4 # via # flyteidl # flytekit + # googleapis-common-protos + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl py==1.11.0 # via retry pyarrow==6.0.1 @@ -109,7 +121,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -117,7 +129,7 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.1.18 +regex==2022.3.2 # via docker-image-py requests==2.27.1 # via @@ -128,6 +140,8 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -139,7 +153,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-greatexpectations/requirements.txt b/plugins/flytekit-greatexpectations/requirements.txt index 7aa0a867dd..31b3be4b48 100644 --- a/plugins/flytekit-greatexpectations/requirements.txt +++ b/plugins/flytekit-greatexpectations/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in @@ -8,10 +8,6 @@ # via -r requirements.in altair==4.2.0 # via great-expectations -appnope==0.1.2 - # via - # ipykernel - # ipython argon2-cffi==21.3.0 # via notebook argon2-cffi-bindings==21.2.0 @@ -26,23 +22,22 @@ backcall==0.2.0 # via ipython binaryornot==0.4.4 # via cookiecutter -black==21.12b0 - # via ipython bleach==4.1.0 # via nbconvert certifi==2021.10.8 # via requests cffi==1.15.0 - # via argon2-cffi-bindings + # via + # argon2-cffi-bindings + # cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.10 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.4 # via - # black # cookiecutter # flytekit # great-expectations @@ -50,8 +45,10 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.4 # via flytekit +cryptography==36.0.1 + # via secretstorage dataclasses-json==0.5.6 # via flytekit debugpy==1.5.1 @@ -70,34 +67,36 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -entrypoints==0.3 +entrypoints==0.4 # via # altair # jupyter-client # nbconvert -executing==0.8.2 +executing==0.8.3 # via stack-data -flyteidl==0.21.24 +flyteidl==0.23.0 # via flytekit -flytekit==0.26.1 +flytekit==0.30.3 # via flytekitplugins-great-expectations -great-expectations==0.14.3 +googleapis-common-protos==1.55.0 + # via flyteidl +great-expectations==0.14.5 # via flytekitplugins-great-expectations greenlet==1.1.2 # via sqlalchemy -grpcio==1.43.0 +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.2 # via # great-expectations # keyring -ipykernel==6.7.0 +ipykernel==6.9.1 # via # ipywidgets # notebook -ipython==8.0.1 +ipython==8.1.0 # via # ipykernel # ipywidgets @@ -110,6 +109,10 @@ ipywidgets==7.6.5 # via great-expectations jedi==0.18.1 # via ipython +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # altair @@ -134,7 +137,7 @@ jupyter-client==7.1.2 # ipykernel # nbclient # notebook -jupyter-core==4.9.1 +jupyter-core==4.9.2 # via # jupyter-client # nbconvert @@ -146,7 +149,7 @@ jupyterlab-widgets==1.0.2 # via ipywidgets keyring==23.5.0 # via flytekit -markupsafe==2.0.1 +markupsafe==2.1.0 # via jinja2 marshmallow==3.14.1 # via @@ -166,14 +169,12 @@ mistune==0.8.4 # great-expectations # nbconvert mypy-extensions==0.4.3 - # via - # black - # typing-inspect -natsort==8.0.2 + # via typing-inspect +natsort==8.1.0 # via flytekit -nbclient==0.5.10 +nbclient==0.5.11 # via nbconvert -nbconvert==6.4.1 +nbconvert==6.4.2 # via notebook nbformat==5.1.3 # via @@ -189,7 +190,7 @@ nest-asyncio==1.5.4 # notebook notebook==6.4.8 # via widgetsnbextension -numpy==1.22.1 +numpy==1.22.2 # via # altair # great-expectations @@ -198,7 +199,7 @@ numpy==1.22.1 # scipy packaging==21.3 # via bleach -pandas==1.4.0 +pandas==1.4.1 # via # altair # flytekit @@ -207,24 +208,24 @@ pandocfilters==1.5.0 # via nbconvert parso==0.8.3 # via jedi -pathspec==0.9.0 - # via black pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -platformdirs==2.4.1 - # via black poyo==0.5.0 # via cookiecutter prometheus-client==0.13.1 # via notebook -prompt-toolkit==3.0.26 +prompt-toolkit==3.0.28 # via ipython protobuf==3.19.4 # via # flyteidl # flytekit + # googleapis-common-protos + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl ptyprocess==0.7.0 # via # pexpect @@ -248,7 +249,7 @@ pyparsing==2.4.7 # packaging pyrsistent==0.18.1 # via jsonschema -python-dateutil==2.8.1 +python-dateutil==2.8.2 # via # arrow # croniter @@ -258,7 +259,7 @@ python-dateutil==2.8.1 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -273,7 +274,7 @@ pyzmq==22.3.0 # via # jupyter-client # notebook -regex==2022.1.18 +regex==2022.3.2 # via docker-image-py requests==2.27.1 # via @@ -281,16 +282,16 @@ requests==2.27.1 # flytekit # great-expectations # responses -responses==0.17.0 +responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit -ruamel-yaml==0.17.17 +ruamel.yaml==0.17.17 # via great-expectations -ruamel-yaml-clib==0.2.6 - # via ruamel-yaml -scipy==1.7.3 +scipy==1.8.0 # via great-expectations +secretstorage==3.3.1 + # via keyring send2trash==1.8.0 # via notebook six==1.16.0 @@ -298,17 +299,15 @@ six==1.16.0 # asttokens # bleach # cookiecutter - # flytekit # grpcio # python-dateutil - # responses sortedcontainers==2.4.0 # via flytekit sqlalchemy==1.4.31 # via # -r requirements.in # flytekitplugins-great-expectations -stack-data==0.1.4 +stack-data==0.2.0 # via ipython statsd==3.3.0 # via flytekit @@ -316,12 +315,10 @@ termcolor==1.1.0 # via great-expectations terminado==0.13.1 # via notebook -testpath==0.5.0 +testpath==0.6.0 # via nbconvert text-unidecode==1.3 # via python-slugify -tomli==1.2.3 - # via black toolz==0.11.2 # via altair tornado==6.1 @@ -330,7 +327,7 @@ tornado==6.1 # jupyter-client # notebook # terminado -tqdm==4.62.3 +tqdm==4.63.0 # via great-expectations traitlets==5.1.1 # via @@ -344,9 +341,10 @@ traitlets==5.1.1 # nbconvert # nbformat # notebook -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via - # black + # flytekit + # great-expectations # typing-inspect typing-inspect==0.7.1 # via dataclasses-json diff --git a/plugins/flytekit-hive/requirements.txt b/plugins/flytekit-hive/requirements.txt index 3ae057f9a0..69ed1e5f3e 100644 --- a/plugins/flytekit-hive/requirements.txt +++ b/plugins/flytekit-hive/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in @@ -12,13 +12,15 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.4 # via # cookiecutter # flytekit @@ -26,7 +28,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.4 # via flytekit cryptography==36.0.1 # via secretstorage @@ -42,16 +44,22 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 +flyteidl==0.23.0 # via flytekit -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-hive -grpcio==1.43.0 +googleapis-common-protos==1.55.0 + # via flyteidl +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.2 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -60,7 +68,7 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -markupsafe==2.0.1 +markupsafe==2.1.0 # via jinja2 marshmallow==3.14.1 # via @@ -79,7 +87,7 @@ numpy==1.22.2 # via # pandas # pyarrow -pandas==1.4.0 +pandas==1.4.1 # via flytekit poyo==0.5.0 # via cookiecutter @@ -87,6 +95,10 @@ protobuf==3.19.4 # via # flyteidl # flytekit + # googleapis-common-protos + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl py==1.11.0 # via retry pyarrow==6.0.1 @@ -101,7 +113,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -109,7 +121,7 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.1.18 +regex==2022.3.2 # via docker-image-py requests==2.27.1 # via @@ -120,6 +132,8 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -131,7 +145,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-k8s-pod/requirements.txt b/plugins/flytekit-k8s-pod/requirements.txt index d60d9c46b6..df398913af 100644 --- a/plugins/flytekit-k8s-pod/requirements.txt +++ b/plugins/flytekit-k8s-pod/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in @@ -16,13 +16,15 @@ certifi==2021.10.8 # via # kubernetes # requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.4 # via # cookiecutter # flytekit @@ -30,7 +32,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.4 # via flytekit cryptography==36.0.1 # via secretstorage @@ -46,18 +48,24 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 +flyteidl==0.23.0 # via flytekit -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-pod google-auth==2.6.0 # via kubernetes -grpcio==1.43.0 +googleapis-common-protos==1.55.0 + # via flyteidl +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.2 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -66,9 +74,9 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -kubernetes==21.7.0 +kubernetes==23.3.0 # via flytekitplugins-pod -markupsafe==2.0.1 +markupsafe==2.1.0 # via jinja2 marshmallow==3.14.1 # via @@ -89,7 +97,7 @@ numpy==1.22.2 # pyarrow oauthlib==3.2.0 # via requests-oauthlib -pandas==1.4.0 +pandas==1.4.1 # via flytekit poyo==0.5.0 # via cookiecutter @@ -97,6 +105,10 @@ protobuf==3.19.4 # via # flyteidl # flytekit + # googleapis-common-protos + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl py==1.11.0 # via retry pyarrow==6.0.1 @@ -118,7 +130,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -128,7 +140,7 @@ pytz==2021.3 # pandas pyyaml==6.0 # via kubernetes -regex==2022.1.18 +regex==2022.3.2 # via docker-image-py requests==2.27.1 # via @@ -145,6 +157,8 @@ retry==0.9.2 # via flytekit rsa==4.8 # via google-auth +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -158,7 +172,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect @@ -170,7 +184,7 @@ urllib3==1.26.8 # kubernetes # requests # responses -websocket-client==1.2.3 +websocket-client==1.3.1 # via kubernetes wheel==0.37.1 # via flytekit diff --git a/plugins/flytekit-kf-mpi/requirements.txt b/plugins/flytekit-kf-mpi/requirements.txt index c13f9f3b74..82a2317f2f 100644 --- a/plugins/flytekit-kf-mpi/requirements.txt +++ b/plugins/flytekit-kf-mpi/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in @@ -12,13 +12,15 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.4 # via # cookiecutter # flytekit @@ -26,7 +28,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.4 # via flytekit cryptography==36.0.1 # via secretstorage @@ -42,18 +44,24 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 +flyteidl==0.23.0 # via # flytekit # flytekitplugins-kfmpi -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-kfmpi -grpcio==1.43.0 +googleapis-common-protos==1.55.0 + # via flyteidl +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.2 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -62,7 +70,7 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -markupsafe==2.0.1 +markupsafe==2.1.0 # via jinja2 marshmallow==3.14.1 # via @@ -81,7 +89,7 @@ numpy==1.22.2 # via # pandas # pyarrow -pandas==1.4.0 +pandas==1.4.1 # via flytekit poyo==0.5.0 # via cookiecutter @@ -89,6 +97,10 @@ protobuf==3.19.4 # via # flyteidl # flytekit + # googleapis-common-protos + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl py==1.11.0 # via retry pyarrow==6.0.1 @@ -103,7 +115,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -111,7 +123,7 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.1.18 +regex==2022.3.2 # via docker-image-py requests==2.27.1 # via @@ -122,6 +134,8 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -133,7 +147,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-kf-pytorch/requirements.txt b/plugins/flytekit-kf-pytorch/requirements.txt index 1696646c87..0a53a62333 100644 --- a/plugins/flytekit-kf-pytorch/requirements.txt +++ b/plugins/flytekit-kf-pytorch/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in @@ -12,13 +12,15 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.4 # via # cookiecutter # flytekit @@ -26,7 +28,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.4 # via flytekit cryptography==36.0.1 # via secretstorage @@ -42,16 +44,22 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 +flyteidl==0.23.0 # via flytekit -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-kfpytorch -grpcio==1.43.0 +googleapis-common-protos==1.55.0 + # via flyteidl +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.2 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -60,7 +68,7 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -markupsafe==2.0.1 +markupsafe==2.1.0 # via jinja2 marshmallow==3.14.1 # via @@ -79,7 +87,7 @@ numpy==1.22.2 # via # pandas # pyarrow -pandas==1.4.0 +pandas==1.4.1 # via flytekit poyo==0.5.0 # via cookiecutter @@ -87,6 +95,10 @@ protobuf==3.19.4 # via # flyteidl # flytekit + # googleapis-common-protos + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl py==1.11.0 # via retry pyarrow==6.0.1 @@ -101,7 +113,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -109,7 +121,7 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.1.18 +regex==2022.3.2 # via docker-image-py requests==2.27.1 # via @@ -120,6 +132,8 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -131,7 +145,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-kf-tensorflow/requirements.txt b/plugins/flytekit-kf-tensorflow/requirements.txt index d3b6253664..b782c39630 100644 --- a/plugins/flytekit-kf-tensorflow/requirements.txt +++ b/plugins/flytekit-kf-tensorflow/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in @@ -12,13 +12,15 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.4 # via # cookiecutter # flytekit @@ -26,7 +28,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.4 # via flytekit cryptography==36.0.1 # via secretstorage @@ -42,16 +44,22 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 +flyteidl==0.23.0 # via flytekit -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-kftensorflow -grpcio==1.43.0 +googleapis-common-protos==1.55.0 + # via flyteidl +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.2 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -60,7 +68,7 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -markupsafe==2.0.1 +markupsafe==2.1.0 # via jinja2 marshmallow==3.14.1 # via @@ -79,7 +87,7 @@ numpy==1.22.2 # via # pandas # pyarrow -pandas==1.4.0 +pandas==1.4.1 # via flytekit poyo==0.5.0 # via cookiecutter @@ -87,6 +95,10 @@ protobuf==3.19.4 # via # flyteidl # flytekit + # googleapis-common-protos + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl py==1.11.0 # via retry pyarrow==6.0.1 @@ -101,7 +113,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -109,7 +121,7 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.1.18 +regex==2022.3.2 # via docker-image-py requests==2.27.1 # via @@ -120,6 +132,8 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -131,7 +145,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-pandera/requirements.txt b/plugins/flytekit-pandera/requirements.txt index f0b65cec17..7b72a0b4c0 100644 --- a/plugins/flytekit-pandera/requirements.txt +++ b/plugins/flytekit-pandera/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in @@ -12,13 +12,15 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.4 # via # cookiecutter # flytekit @@ -26,7 +28,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.4 # via flytekit cryptography==36.0.1 # via secretstorage @@ -42,16 +44,22 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 +flyteidl==0.23.0 # via flytekit -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-pandera -grpcio==1.43.0 +googleapis-common-protos==1.55.0 + # via flyteidl +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.2 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -60,7 +68,7 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -markupsafe==2.0.1 +markupsafe==2.1.0 # via jinja2 marshmallow==3.14.1 # via @@ -82,11 +90,11 @@ numpy==1.22.2 # pyarrow packaging==21.3 # via pandera -pandas==1.4.0 +pandas==1.4.1 # via # flytekit # pandera -pandera==0.8.1 +pandera==0.9.0 # via flytekitplugins-pandera poyo==0.5.0 # via cookiecutter @@ -94,6 +102,10 @@ protobuf==3.19.4 # via # flyteidl # flytekit + # googleapis-common-protos + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl py==1.11.0 # via retry pyarrow==6.0.1 @@ -102,6 +114,8 @@ pyarrow==6.0.1 # pandera pycparser==2.21 # via cffi +pydantic==1.9.0 + # via pandera pyparsing==3.0.7 # via packaging python-dateutil==2.8.2 @@ -112,7 +126,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -120,7 +134,7 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.1.18 +regex==2022.3.2 # via docker-image-py requests==2.27.1 # via @@ -131,6 +145,8 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -142,9 +158,10 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit + # pydantic # typing-inspect typing-inspect==0.7.1 # via diff --git a/plugins/flytekit-papermill/dev-requirements.in b/plugins/flytekit-papermill/dev-requirements.in index 32404e0a2d..7a47470868 100644 --- a/plugins/flytekit-papermill/dev-requirements.in +++ b/plugins/flytekit-papermill/dev-requirements.in @@ -1,2 +1,3 @@ -git+https://github.com/flyteorg/flytekit@add-sd-make-class-methods#egg=flytekitplugins-spark&subdirectory=plugins/flytekit-spark +flyteidl>=0.23.0 +git+https://github.com/flyteorg/flytekit@offloaded-objects-caching#egg=flytekitplugins-spark&subdirectory=plugins/flytekit-spark # vcs+protocol://repo_url/#egg=pkg&subdirectory=flyte diff --git a/plugins/flytekit-papermill/dev-requirements.txt b/plugins/flytekit-papermill/dev-requirements.txt index ef30545338..f777991cc3 100644 --- a/plugins/flytekit-papermill/dev-requirements.txt +++ b/plugins/flytekit-papermill/dev-requirements.txt @@ -10,6 +10,8 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot charset-normalizer==2.0.10 @@ -26,6 +28,8 @@ cookiecutter==1.7.3 # via flytekit croniter==1.2.0 # via flytekit +cryptography==36.0.1 + # via secretstorage dataclasses-json==0.5.6 # via flytekit decorator==5.1.1 @@ -38,18 +42,26 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 - # via flytekit +flyteidl==0.23.0 + # via + # -r dev-requirements.in + # flytekit flytekit==0.30.0 # via flytekitplugins-spark -flytekitplugins-spark @ git+https://github.com/flyteorg/flytekit@add-sd-make-class-methods#subdirectory=plugins/flytekit-spark +flytekitplugins-spark @ git+https://github.com/flyteorg/flytekit@offloaded-objects-caching#subdirectory=plugins/flytekit-spark # via -r dev-requirements.in +googleapis-common-protos==1.55.0 + # via flyteidl grpcio==1.43.0 # via flytekit idna==3.3 # via requests importlib-metadata==4.10.1 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -85,12 +97,18 @@ protobuf==3.19.3 # via # flyteidl # flytekit + # googleapis-common-protos + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl py==1.11.0 # via retry py4j==0.10.9.3 # via pyspark pyarrow==6.0.1 # via flytekit +pycparser==2.21 + # via cffi pyspark==3.2.1 # via flytekitplugins-spark python-dateutil==2.8.1 @@ -120,6 +138,8 @@ responses==0.17.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter diff --git a/plugins/flytekit-snowflake/requirements.txt b/plugins/flytekit-snowflake/requirements.txt index 53951100b5..2d34ec966f 100644 --- a/plugins/flytekit-snowflake/requirements.txt +++ b/plugins/flytekit-snowflake/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in @@ -12,13 +12,15 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.4 # via # cookiecutter # flytekit @@ -26,7 +28,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.4 # via flytekit cryptography==36.0.1 # via secretstorage @@ -42,16 +44,22 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 +flyteidl==0.23.0 # via flytekit -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-snowflake -grpcio==1.43.0 +googleapis-common-protos==1.55.0 + # via flyteidl +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.2 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -60,7 +68,7 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -markupsafe==2.0.1 +markupsafe==2.1.0 # via jinja2 marshmallow==3.14.1 # via @@ -79,7 +87,7 @@ numpy==1.22.2 # via # pandas # pyarrow -pandas==1.4.0 +pandas==1.4.1 # via flytekit poyo==0.5.0 # via cookiecutter @@ -87,6 +95,10 @@ protobuf==3.19.4 # via # flyteidl # flytekit + # googleapis-common-protos + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl py==1.11.0 # via retry pyarrow==6.0.1 @@ -101,7 +113,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -109,7 +121,7 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.1.18 +regex==2022.3.2 # via docker-image-py requests==2.27.1 # via @@ -120,6 +132,8 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -131,7 +145,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-spark/requirements.txt b/plugins/flytekit-spark/requirements.txt index 75d1e32d53..6e7c6e59f0 100644 --- a/plugins/flytekit-spark/requirements.txt +++ b/plugins/flytekit-spark/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in @@ -12,13 +12,15 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.4 # via # cookiecutter # flytekit @@ -26,7 +28,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.4 # via flytekit cryptography==36.0.1 # via secretstorage @@ -42,16 +44,22 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 +flyteidl==0.23.0 # via flytekit -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-spark -grpcio==1.43.0 +googleapis-common-protos==1.55.0 + # via flyteidl +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.2 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -60,7 +68,7 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -markupsafe==2.0.1 +markupsafe==2.1.0 # via jinja2 marshmallow==3.14.1 # via @@ -79,7 +87,7 @@ numpy==1.22.2 # via # pandas # pyarrow -pandas==1.4.0 +pandas==1.4.1 # via flytekit poyo==0.5.0 # via cookiecutter @@ -87,6 +95,10 @@ protobuf==3.19.4 # via # flyteidl # flytekit + # googleapis-common-protos + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl py==1.11.0 # via retry py4j==0.10.9.3 @@ -105,7 +117,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -113,7 +125,7 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.1.18 +regex==2022.3.2 # via docker-image-py requests==2.27.1 # via @@ -124,6 +136,8 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -135,7 +149,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-sqlalchemy/requirements.txt b/plugins/flytekit-sqlalchemy/requirements.txt index 93dcc2790c..e617920549 100644 --- a/plugins/flytekit-sqlalchemy/requirements.txt +++ b/plugins/flytekit-sqlalchemy/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in @@ -12,13 +12,15 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.4 # via # cookiecutter # flytekit @@ -26,7 +28,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.4 # via flytekit cryptography==36.0.1 # via secretstorage @@ -42,18 +44,24 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 +flyteidl==0.23.0 # via flytekit -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-sqlalchemy +googleapis-common-protos==1.55.0 + # via flyteidl greenlet==1.1.2 # via sqlalchemy -grpcio==1.43.0 +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.2 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -62,7 +70,7 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -markupsafe==2.0.1 +markupsafe==2.1.0 # via jinja2 marshmallow==3.14.1 # via @@ -81,7 +89,7 @@ numpy==1.22.2 # via # pandas # pyarrow -pandas==1.4.0 +pandas==1.4.1 # via flytekit poyo==0.5.0 # via cookiecutter @@ -89,6 +97,10 @@ protobuf==3.19.4 # via # flyteidl # flytekit + # googleapis-common-protos + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl py==1.11.0 # via retry pyarrow==6.0.1 @@ -103,7 +115,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -111,7 +123,7 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.1.18 +regex==2022.3.2 # via docker-image-py requests==2.27.1 # via @@ -122,6 +134,8 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -135,7 +149,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect diff --git a/requirements-spark2.txt b/requirements-spark2.txt index 6fd2069c47..be50ade576 100644 --- a/requirements-spark2.txt +++ b/requirements-spark2.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.8 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # make requirements-spark2.txt @@ -18,6 +18,8 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot charset-normalizer==2.0.12 @@ -34,6 +36,8 @@ cookiecutter==1.7.3 # via flytekit croniter==1.3.4 # via flytekit +cryptography==36.0.1 + # via secretstorage dataclasses-json==0.5.6 # via flytekit decorator==5.1.1 @@ -46,16 +50,20 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.3 +flyteidl==0.23.0 # via flytekit -googleapis-common-protos==1.54.0 +googleapis-common-protos==1.55.0 # via flyteidl grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.11.1 +importlib-metadata==4.11.2 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -104,6 +112,8 @@ py==1.11.0 # via retry pyarrow==6.0.1 # via flytekit +pycparser==2.21 + # via cffi pyrsistent==0.18.1 # via jsonschema python-dateutil==2.8.2 @@ -114,7 +124,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==6.1.0 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -135,6 +145,8 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter diff --git a/requirements.txt b/requirements.txt index 0fc2b924d9..2e51986bf4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.8 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # make requirements.txt @@ -16,6 +16,8 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot charset-normalizer==2.0.12 @@ -32,6 +34,8 @@ cookiecutter==1.7.3 # via flytekit croniter==1.3.4 # via flytekit +cryptography==36.0.1 + # via secretstorage dataclasses-json==0.5.6 # via flytekit decorator==5.1.1 @@ -44,16 +48,20 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.3 +flyteidl==0.23.0 # via flytekit -googleapis-common-protos==1.54.0 +googleapis-common-protos==1.55.0 # via flyteidl grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.11.1 +importlib-metadata==4.11.2 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -102,6 +110,8 @@ py==1.11.0 # via retry pyarrow==6.0.1 # via flytekit +pycparser==2.21 + # via cffi pyrsistent==0.18.1 # via jsonschema python-dateutil==2.8.2 @@ -112,7 +122,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==6.1.0 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -133,6 +143,8 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter diff --git a/setup.py b/setup.py index 4caa974c59..9a96657326 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ ] }, install_requires=[ - "flyteidl>=0.22.3", + "flyteidl>=0.23.0", "wheel>=0.30.0,<1.0.0", "pandas>=1.0.0,<2.0.0", "pyarrow>=4.0.0,<7.0.0", diff --git a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt index eb7db6ee4b..15a1f89700 100644 --- a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt +++ b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.8 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # make tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt @@ -10,6 +10,8 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot charset-normalizer==2.0.12 @@ -26,6 +28,8 @@ cookiecutter==1.7.3 # via flytekit croniter==1.3.4 # via flytekit +cryptography==36.0.1 + # via secretstorage cycler==0.11.0 # via matplotlib dataclasses-json==0.5.6 @@ -40,20 +44,24 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.3 +flyteidl==0.23.0 # via flytekit flytekit==0.30.3 # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in fonttools==4.29.1 # via matplotlib -googleapis-common-protos==1.54.0 +googleapis-common-protos==1.55.0 # via flyteidl grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.11.1 +importlib-metadata==4.11.2 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -111,19 +119,22 @@ py==1.11.0 # via retry pyarrow==6.0.1 # via flytekit +pycparser==2.21 + # via cffi pyparsing==3.0.7 # via # matplotlib # packaging python-dateutil==2.8.2 # via + # arrow # croniter # flytekit # matplotlib # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==6.1.0 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -142,6 +153,8 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter diff --git a/tests/flytekit/unit/core/test_local_cache.py b/tests/flytekit/unit/core/test_local_cache.py index bf4e4c2edb..3f3e56de88 100644 --- a/tests/flytekit/unit/core/test_local_cache.py +++ b/tests/flytekit/unit/core/test_local_cache.py @@ -1,12 +1,15 @@ import datetime import typing from dataclasses import dataclass +from typing import List import pandas from dataclasses_json import dataclass_json from pytest import fixture +from typing_extensions import Annotated -from flytekit import SQLTask, kwtypes +from flytekit import SQLTask, dynamic, kwtypes +from flytekit.core.hash import HashMethod from flytekit.core.local_cache import LocalTaskCache from flytekit.core.task import TaskMetadata, task from flytekit.core.testing import task_mock @@ -252,13 +255,131 @@ def my_wf(a: int, b: str) -> (int, str): assert n_cached_task_calls == 2 -""" -Update SD transformer so that it can to_python_value a Schema literal - - If a Schema literal is detected, copy the uri and use the new decoder to unwrap the uri -Update FS transformer so that it can to_python_value a StructuredDataset literal - - If a StructuredDataset literal is detected, use the uri from that instead. +def test_set_integer_literal_hash_is_not_cached(): + """ + Test to confirm that the local cache is not set in the case of integers, even if we + return an annotated integer. In order to make this very explicit, we define a constant hash + function, i.e. the same value is returned by it regardless of the input. + """ + + def constant_hash_function(a: int) -> str: + return "hash" + + @task + def t0(a: int) -> Annotated[int, HashMethod(function=constant_hash_function)]: + return a + + @task(cache=True, cache_version="0.0.1") + def t1(cached_a: int) -> int: + global n_cached_task_calls + n_cached_task_calls += 1 + return cached_a + + @workflow + def wf(a: int) -> int: + annotated_a = t0(a=a) + return t1(cached_a=annotated_a) + + assert n_cached_task_calls == 0 + assert wf(a=3) == 3 + assert n_cached_task_calls == 1 + # Confirm that the value is not cached, even though we set a hash function that + # returns a constant value and that the task has only one input. + assert wf(a=2) == 2 + assert n_cached_task_calls == 2 + # Confirm that the cache is hit if we execute the workflow with the same value as previous run. + assert wf(a=2) == 2 + assert n_cached_task_calls == 2 + + +def test_pass_annotated_to_downstream_tasks(): + @task + def t0(a: int) -> Annotated[int, HashMethod(function=str)]: + return a + 1 + + @task(cache=True, cache_version="42") + def downstream_t(a: int) -> int: + global n_cached_task_calls + n_cached_task_calls += 1 + return a + 2 + + @dynamic + def t1(a: int) -> int: + v = t0(a=a) + + # We should have a cache miss in the first call to downstream_t and have a cache hit + # on the second call. + v_1 = downstream_t(a=v) + v_2 = downstream_t(a=v) + + return v_1 + v_2 + + assert n_cached_task_calls == 0 + assert t1(a=3) == (6 + 6) + assert n_cached_task_calls == 1 + + +def test_pandas_dataframe_hash(): + """ + Test that cache is hit in the case of pandas dataframes where we annotated dataframes to hash + the contents of the dataframes. + """ + + def hash_pandas_dataframe(df: pandas.DataFrame) -> str: + return str(pandas.util.hash_pandas_object(df)) + + @task + def uncached_data_reading_task() -> Annotated[pandas.DataFrame, HashMethod(hash_pandas_dataframe)]: + return pandas.DataFrame({"column_1": [1, 2, 3]}) + + @task(cache=True, cache_version="0.1") + def cached_data_processing_task(data: pandas.DataFrame) -> pandas.DataFrame: + global n_cached_task_calls + n_cached_task_calls += 1 + return data * 2 + + @workflow + def my_workflow(): + raw_data = uncached_data_reading_task() + cached_data_processing_task(data=raw_data) + + assert n_cached_task_calls == 0 + my_workflow() + assert n_cached_task_calls == 1 + + # Confirm that we see a cache hit in the case of annotated dataframes. + my_workflow() + assert n_cached_task_calls == 1 -Update all plugins that can take in a FlyteSchema to also be able to take in a StructuredDataset. -All tests should work with the presence of SD imports. -""" +def test_list_of_pandas_dataframe_hash(): + """ + Test that cache is hit in the case of a list of pandas dataframes where we annotated dataframes to hash + the contents of the dataframes. + """ + + def hash_pandas_dataframe(df: pandas.DataFrame) -> str: + return str(pandas.util.hash_pandas_object(df)) + + @task + def uncached_data_reading_task() -> List[Annotated[pandas.DataFrame, HashMethod(hash_pandas_dataframe)]]: + return [pandas.DataFrame({"column_1": [1, 2, 3]}), pandas.DataFrame({"column_1": [10, 20, 30]})] + + @task(cache=True, cache_version="0.1") + def cached_data_processing_task(data: List[pandas.DataFrame]) -> List[pandas.DataFrame]: + global n_cached_task_calls + n_cached_task_calls += 1 + return [df * 2 for df in data] + + @workflow + def my_workflow(): + raw_data = uncached_data_reading_task() + cached_data_processing_task(data=raw_data) + + assert n_cached_task_calls == 0 + my_workflow() + assert n_cached_task_calls == 1 + + # Confirm that we see a cache hit in the case of annotated dataframes. + my_workflow() + assert n_cached_task_calls == 1 diff --git a/tests/flytekit/unit/core/test_type_engine.py b/tests/flytekit/unit/core/test_type_engine.py index df1ae3e11f..56f3ca50af 100644 --- a/tests/flytekit/unit/core/test_type_engine.py +++ b/tests/flytekit/unit/core/test_type_engine.py @@ -17,10 +17,14 @@ from marshmallow_enum import LoadDumpOptions from marshmallow_jsonschema import JSONSchema from pandas._testing import assert_frame_equal +from typing_extensions import Annotated from flytekit import kwtypes from flytekit.core.annotation import FlyteAnnotation from flytekit.core.context_manager import FlyteContext, FlyteContextManager +from flytekit.core.dynamic_workflow_task import dynamic +from flytekit.core.hash import HashMethod +from flytekit.core.task import task from flytekit.core.type_engine import ( DataclassTransformer, DictTransformer, @@ -44,13 +48,9 @@ from flytekit.types.pickle import FlytePickle from flytekit.types.pickle.pickle import FlytePickleTransformer from flytekit.types.schema import FlyteSchema +from flytekit.types.schema.types_pandas import PandasDataFrameTransformer from flytekit.types.structured.structured_dataset import StructuredDataset -try: - from typing import Annotated -except ImportError: - from typing_extensions import Annotated - def test_type_engine(): t = int @@ -843,6 +843,77 @@ def test_dict_to_literal_map_with_wrong_input_type(): TypeEngine.dict_to_literal_map(ctx, input, guessed_python_types) +def test_pass_annotated_to_downstream_tasks(): + """ + Test to confirm that the loaded dataframe is not affected and can be used in @dynamic. + """ + # pandas dataframe hash function + def hash_pandas_dataframe(df: pd.DataFrame) -> str: + return str(pd.util.hash_pandas_object(df)) + + @task + def t0(a: int) -> Annotated[int, HashMethod(function=str)]: + return a + 1 + + @task + def annotated_return_task() -> Annotated[pd.DataFrame, HashMethod(hash_pandas_dataframe)]: + return pd.DataFrame({"column_1": [1, 2, 3]}) + + @task(cache=True, cache_version="42") + def downstream_t(a: int, df: pd.DataFrame) -> int: + return a + 2 + len(df) + + @dynamic + def t1(a: int) -> int: + v = t0(a=a) + df = annotated_return_task() + + # We should have a cache miss in the first call to downstream_t + v_1 = downstream_t(a=v, df=df) + v_2 = downstream_t(a=v, df=df) + + return v_1 + v_2 + + assert t1(a=3) == (6 + 6 + 6) + + +def test_literal_hash_int_not_set(): + """ + Test to confirm that annotating an integer with `HashMethod` does not force the literal to have its + hash set. + """ + ctx = FlyteContext.current_context() + lv = TypeEngine.to_literal( + ctx, 42, Annotated[int, HashMethod(str)], LiteralType(simple=model_types.SimpleType.INTEGER) + ) + assert lv.scalar.primitive.integer == 42 + assert lv.hash is None + + +def test_literal_hash_to_python_value(): + """ + Test to confirm that literals can be converted to python values, regardless of the hash value set in the literal. + """ + ctx = FlyteContext.current_context() + + def constant_hash(df: pd.DataFrame) -> str: + return "h4Sh" + + df = pd.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}) + pandas_df_transformer = PandasDataFrameTransformer() + literal_with_hash_set = TypeEngine.to_literal( + ctx, + df, + Annotated[pd.DataFrame, HashMethod(constant_hash)], + pandas_df_transformer.get_literal_type(pd.DataFrame), + ) + assert literal_with_hash_set.hash == "h4Sh" + # Confirm tha the loaded dataframe is not affected + python_df = TypeEngine.to_python_value(ctx, literal_with_hash_set, pd.DataFrame) + expected_df = pd.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}) + assert expected_df.equals(python_df) + + def test_annotated_simple_types(): def _check_annotation(t, annotation): lt = TypeEngine.to_literal_type(t) diff --git a/tests/flytekit/unit/core/test_type_hints.py b/tests/flytekit/unit/core/test_type_hints.py index a2508ce5de..e5ed338599 100644 --- a/tests/flytekit/unit/core/test_type_hints.py +++ b/tests/flytekit/unit/core/test_type_hints.py @@ -14,6 +14,7 @@ from dataclasses_json import dataclass_json from google.protobuf.struct_pb2 import Struct from pandas._testing import assert_frame_equal +from typing_extensions import Annotated import flytekit from flytekit import ContainerTask, Secret, SQLTask, dynamic, kwtypes, map_task @@ -21,6 +22,7 @@ from flytekit.core.condition import conditional from flytekit.core.context_manager import ExecutionState, FastSerializationSettings, Image, ImageConfig from flytekit.core.data_persistence import FileAccessProvider +from flytekit.core.hash import HashMethod from flytekit.core.node import Node from flytekit.core.promise import NodeOutput, Promise, VoidPromise from flytekit.core.resources import Resources @@ -1599,3 +1601,110 @@ def foo3(a: typing.Dict) -> typing.Dict: with pytest.raises(TypeError, match="Not a collection type simple: STRUCT\n but got a list \\[{'hello': 2}\\]"): foo3(a=[{"hello": 2}]) + + +def test_task_annotate_primitive_type_has_no_effect(): + @task + def plus_two( + a: int, + ) -> Annotated[int, HashMethod(str)]: # Note the use of `str` as the hash function for ints. This has no effect. + return a + 2 + + assert plus_two(a=1) == 3 + + ctx = context_manager.FlyteContextManager.current_context() + output_lm = plus_two.dispatch_execute( + ctx, + _literal_models.LiteralMap( + literals={ + "a": _literal_models.Literal( + scalar=_literal_models.Scalar(primitive=_literal_models.Primitive(integer=3)) + ) + } + ), + ) + assert output_lm.literals["o0"].scalar.primitive.integer == 5 + assert output_lm.literals["o0"].hash is None + + +def test_task_hash_return_pandas_dataframe(): + constant_value = "road-hash" + + def constant_function(df: pandas.DataFrame) -> str: + return constant_value + + @task + def t0() -> Annotated[pandas.DataFrame, HashMethod(constant_function)]: + return pandas.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}) + + ctx = context_manager.FlyteContextManager.current_context() + output_lm = t0.dispatch_execute(ctx, _literal_models.LiteralMap(literals={})) + assert output_lm.literals["o0"].hash == constant_value + + # Confirm that the literal containing a hash does not have any effect on the scalar. + df = TypeEngine.to_python_value(ctx, output_lm.literals["o0"], pandas.DataFrame) + expected_df = pandas.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}) + assert df.equals(expected_df) + + +def test_workflow_containing_multiple_annotated_tasks(): + def hash_function_t0(df: pandas.DataFrame) -> str: + return "hash-0" + + @task + def t0() -> Annotated[pandas.DataFrame, HashMethod(hash_function_t0)]: + return pandas.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}) + + def hash_function_t1(df: pandas.DataFrame) -> str: + return "hash-1" + + @task + def t1() -> Annotated[pandas.DataFrame, HashMethod(hash_function_t1)]: + return pandas.DataFrame(data={"col1": [10, 20], "col2": [30, 40]}) + + @task + def t2() -> pandas.DataFrame: + return pandas.DataFrame(data={"col1": [100, 200], "col2": [300, 400]}) + + # Auxiliary task used to sum up the dataframes. It demonstrates that the use of `Annotated` does not + # have any impact in the definition and execution of cached or uncached downstream tasks + @task + def sum_dataframes(df0: pandas.DataFrame, df1: pandas.DataFrame, df2: pandas.DataFrame) -> pandas.DataFrame: + return df0 + df1 + df2 + + @workflow + def wf() -> pandas.DataFrame: + df0 = t0() + df1 = t1() + df2 = t2() + return sum_dataframes(df0=df0, df1=df1, df2=df2) + + df = wf() + + expected_df = pandas.DataFrame(data={"col1": [1 + 10 + 100, 2 + 20 + 200], "col2": [3 + 30 + 300, 4 + 40 + 400]}) + assert expected_df.equals(df) + + +def test_list_containing_multiple_annotated_pandas_dataframes(): + def hash_pandas_dataframe(df: pandas.DataFrame) -> str: + return str(pandas.util.hash_pandas_object(df)) + + @task + def produce_list_of_annotated_dataframes() -> typing.List[ + Annotated[pandas.DataFrame, HashMethod(hash_pandas_dataframe)] + ]: + return [pandas.DataFrame({"column_1": [1, 2, 3]}), pandas.DataFrame({"column_1": [4, 5, 6]})] + + @task(cache=True, cache_version="v0") + def sum_list_of_pandas_dataframes(lst: typing.List[pandas.DataFrame]) -> pandas.DataFrame: + return sum(lst) + + @workflow + def wf() -> pandas.DataFrame: + lst = produce_list_of_annotated_dataframes() + return sum_list_of_pandas_dataframes(lst=lst) + + df = wf() + + expected_df = pandas.DataFrame({"column_1": [5, 7, 9]}) + assert expected_df.equals(df) diff --git a/tests/flytekit_compatibility/test_structured_dataset.py b/tests/flytekit_compatibility/test_structured_dataset.py index 20965cb802..f935305210 100644 --- a/tests/flytekit_compatibility/test_structured_dataset.py +++ b/tests/flytekit_compatibility/test_structured_dataset.py @@ -1,8 +1,12 @@ import pandas as pd +from flytekit.configuration.sdk import USE_STRUCTURED_DATASET from flytekit.core.type_engine import TypeEngine def test_pandas_is_schema_with_flag(): + # This test can only be run iff USE_STRUCTURED_DATASET is not set + assert USE_STRUCTURED_DATASET.get() is False + lt = TypeEngine.to_literal_type(pd.DataFrame) assert lt.schema is not None