From 68fc9bb6da49180661afec1c0493c0d540a1670e Mon Sep 17 00:00:00 2001 From: Felix Wang Date: Wed, 15 Sep 2021 13:03:16 -0700 Subject: [PATCH 1/6] Add feature server configuration for AWS lambda Signed-off-by: Felix Wang --- .../infra/feature_servers/aws_lambda/app.py | 23 ++++++++++ sdk/python/feast/repo_config.py | 44 +++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 sdk/python/feast/infra/feature_servers/aws_lambda/app.py diff --git a/sdk/python/feast/infra/feature_servers/aws_lambda/app.py b/sdk/python/feast/infra/feature_servers/aws_lambda/app.py new file mode 100644 index 0000000000..fe1ba000ae --- /dev/null +++ b/sdk/python/feast/infra/feature_servers/aws_lambda/app.py @@ -0,0 +1,23 @@ +from pydantic import StrictBool, StrictStr +from pydantic.typing import Literal + +from feast.repo_config import FeastConfigBaseModel + + +class AwsLambdaFeatureServerConfig(FeastConfigBaseModel): + """Feature server config for AWS Lambda.""" + + type: Literal["aws_lambda"] = "aws_lambda" + """Feature server type selector.""" + + enabled: StrictBool + """Whether the feature server should be launched.""" + + public: StrictBool + """Whether the endpoint should be publicly accessible.""" + + auth: StrictStr + """Authentication method for the endpoint.""" + + execution_role_name: StrictStr + """The execution role for the AWS Lambda function.""" diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index 688fed9578..64a0dc314d 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -32,6 +32,10 @@ "redshift": "feast.infra.offline_stores.redshift.RedshiftOfflineStore", } +FEATURE_SERVER_CONFIG_CLASS_FOR_TYPE = { + "aws_lambda": "feast.infra.feature_servers.aws_lambda.app.AwsLambdaFeatureServerConfig", +} + class FeastBaseModel(BaseModel): """ Feast Pydantic Configuration Class """ @@ -190,6 +194,37 @@ def _validate_offline_store_config(cls, values): return values + @root_validator(pre=True) + def _validate_feature_server_config(cls, values): + # Having no feature server is the default. + if "feature_server" not in values: + return values + + # Make sure that the provider configuration is set. We need it to set the defaults + assert "provider" in values + + # Make sure that the type is not set, since we will set it based on the provider. + assert "type" not in values + + # Set the default type. We only support AWS Lambda for now. + if values["provider"] == "aws": + values["feature_server"]["type"] = "aws_lambda" + + feature_server_type = values["feature_server"]["type"] + + # Validate the dict to ensure one of the union types match + try: + feature_server_config_class = get_feature_server_config_from_type( + feature_server_type + ) + feature_server_config_class(**values["feature_server"]) + except ValidationError as e: + raise ValidationError( + [ErrorWrapper(e, loc="feature_server")], model=RepoConfig, + ) + + return values + @validator("project") def _validate_project_name(cls, v): from feast.repo_operations import is_valid_name @@ -244,6 +279,15 @@ def get_offline_config_from_type(offline_store_type: str): return get_class_from_type(module_name, config_class_name, config_class_name) +def get_feature_server_config_from_type(feature_server_type: str): + # We do not support custom feature servers right now. + assert feature_server_type in FEATURE_SERVER_CONFIG_CLASS_FOR_TYPE + + feature_server_type = FEATURE_SERVER_CONFIG_CLASS_FOR_TYPE[feature_server_type] + module_name, config_class_name = feature_server_type.rsplit(".", 1) + return get_class_from_type(module_name, config_class_name, config_class_name) + + def load_repo_config(repo_path: Path) -> RepoConfig: config_path = repo_path / "feature_store.yaml" From 0bdc6552f019bdd994f654f7ba2b3df777ad1b69 Mon Sep 17 00:00:00 2001 From: Felix Wang Date: Wed, 15 Sep 2021 13:53:53 -0700 Subject: [PATCH 2/6] Rename app.py to config.py and add defaults to config Signed-off-by: Felix Wang --- .../infra/feature_servers/aws_lambda/{app.py => config.py} | 6 +++--- sdk/python/feast/repo_config.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) rename sdk/python/feast/infra/feature_servers/aws_lambda/{app.py => config.py} (84%) diff --git a/sdk/python/feast/infra/feature_servers/aws_lambda/app.py b/sdk/python/feast/infra/feature_servers/aws_lambda/config.py similarity index 84% rename from sdk/python/feast/infra/feature_servers/aws_lambda/app.py rename to sdk/python/feast/infra/feature_servers/aws_lambda/config.py index fe1ba000ae..d026415ec3 100644 --- a/sdk/python/feast/infra/feature_servers/aws_lambda/app.py +++ b/sdk/python/feast/infra/feature_servers/aws_lambda/config.py @@ -10,13 +10,13 @@ class AwsLambdaFeatureServerConfig(FeastConfigBaseModel): type: Literal["aws_lambda"] = "aws_lambda" """Feature server type selector.""" - enabled: StrictBool + enabled: StrictBool = False """Whether the feature server should be launched.""" - public: StrictBool + public: StrictBool = True """Whether the endpoint should be publicly accessible.""" - auth: StrictStr + auth: Literal["none", "api-key"] = "none" """Authentication method for the endpoint.""" execution_role_name: StrictStr diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index 64a0dc314d..0d8c1f8b97 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -33,7 +33,7 @@ } FEATURE_SERVER_CONFIG_CLASS_FOR_TYPE = { - "aws_lambda": "feast.infra.feature_servers.aws_lambda.app.AwsLambdaFeatureServerConfig", + "aws_lambda": "feast.infra.feature_servers.aws_lambda.config.AwsLambdaFeatureServerConfig", } From b27500bdc43211e768c03a751c4a29bfc70590bf Mon Sep 17 00:00:00 2001 From: Felix Wang Date: Wed, 15 Sep 2021 14:17:20 -0700 Subject: [PATCH 3/6] Change asserts to if/raise checks Signed-off-by: Felix Wang --- sdk/python/feast/errors.py | 19 +++++++++++++++++++ sdk/python/feast/repo_config.py | 14 +++++++++++--- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/sdk/python/feast/errors.py b/sdk/python/feast/errors.py index 784e24b043..cf85bdd08c 100644 --- a/sdk/python/feast/errors.py +++ b/sdk/python/feast/errors.py @@ -93,6 +93,25 @@ def __init__(self, provider_name): super().__init__(f"Provider '{provider_name}' is not implemented") +class FeastProviderNotSetError(Exception): + def __init__(self): + super().__init__("Provider is not set, but is required") + + +class FeastFeatureServerTypeSetError(Exception): + def __init__(self, feature_server_type: str): + super().__init__( + f"Feature server type was set to {feature_server_type}, but the type should be determined by the provider" + ) + + +class FeastFeatureServerTypeInvalidError(Exception): + def __init__(self, feature_server_type: str): + super().__init__( + f"Feature server type was set to {feature_server_type}, but this type is invalid" + ) + + class FeastModuleImportError(Exception): def __init__(self, module_name: str, module_type: str): super().__init__(f"Could not import {module_type} module '{module_name}'") diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index 0d8c1f8b97..e3d8a5ad13 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -13,6 +13,11 @@ from pydantic.error_wrappers import ErrorWrapper from pydantic.typing import Dict, Optional, Union +from feast.errors import ( + FeastFeatureServerTypeInvalidError, + FeastFeatureServerTypeSetError, + FeastProviderNotSetError, +) from feast.importer import get_class_from_type from feast.usage import log_exceptions @@ -201,10 +206,12 @@ def _validate_feature_server_config(cls, values): return values # Make sure that the provider configuration is set. We need it to set the defaults - assert "provider" in values + if "provider" not in values: + raise FeastProviderNotSetError() # Make sure that the type is not set, since we will set it based on the provider. - assert "type" not in values + if "type" in values["feature_server"]: + raise FeastFeatureServerTypeSetError(values["feature_server"]["type"]) # Set the default type. We only support AWS Lambda for now. if values["provider"] == "aws": @@ -281,7 +288,8 @@ def get_offline_config_from_type(offline_store_type: str): def get_feature_server_config_from_type(feature_server_type: str): # We do not support custom feature servers right now. - assert feature_server_type in FEATURE_SERVER_CONFIG_CLASS_FOR_TYPE + if feature_server_type not in FEATURE_SERVER_CONFIG_CLASS_FOR_TYPE: + raise FeastFeatureServerTypeInvalidError(feature_server_type) feature_server_type = FEATURE_SERVER_CONFIG_CLASS_FOR_TYPE[feature_server_type] module_name, config_class_name = feature_server_type.rsplit(".", 1) From f868fd0844ccbb12b48d7dd8f7e3566e92396da6 Mon Sep 17 00:00:00 2001 From: Felix Wang Date: Wed, 15 Sep 2021 15:36:30 -0700 Subject: [PATCH 4/6] Always generate feature server config Signed-off-by: Felix Wang --- sdk/python/feast/repo_config.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index e3d8a5ad13..1def2d8674 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -95,6 +95,9 @@ class RepoConfig(FeastBaseModel): offline_store: Any """ OfflineStoreConfig: Offline store configuration (optional depending on provider) """ + feature_server: Any + """ FeatureServerConfig: Feature server configuration (optional depending on provider) """ + repo_path: Optional[Path] = None def __init__(self, **data: Any): @@ -114,6 +117,11 @@ def __init__(self, **data: Any): elif isinstance(self.offline_store, str): self.offline_store = get_offline_config_from_type(self.offline_store)() + if isinstance(self.feature_server, Dict): + self.feature_server = get_feature_server_config_from_type( + self.feature_server["type"] + )(**self.feature_server) + def get_registry_config(self): if isinstance(self.registry, str): return RegistryConfig(path=self.registry) @@ -203,6 +211,7 @@ def _validate_offline_store_config(cls, values): def _validate_feature_server_config(cls, values): # Having no feature server is the default. if "feature_server" not in values: + values["feature_server"] = dict() return values # Make sure that the provider configuration is set. We need it to set the defaults From 8a215b5a2cc47519c5338754ff923e4437351c99 Mon Sep 17 00:00:00 2001 From: Felix Wang Date: Wed, 15 Sep 2021 15:58:19 -0700 Subject: [PATCH 5/6] Make feature server config optional Signed-off-by: Felix Wang --- sdk/python/feast/repo_config.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index 1def2d8674..52853dba9c 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -95,7 +95,7 @@ class RepoConfig(FeastBaseModel): offline_store: Any """ OfflineStoreConfig: Offline store configuration (optional depending on provider) """ - feature_server: Any + feature_server: Optional[Any] """ FeatureServerConfig: Feature server configuration (optional depending on provider) """ repo_path: Optional[Path] = None @@ -211,7 +211,6 @@ def _validate_offline_store_config(cls, values): def _validate_feature_server_config(cls, values): # Having no feature server is the default. if "feature_server" not in values: - values["feature_server"] = dict() return values # Make sure that the provider configuration is set. We need it to set the defaults From 9fb1d0976bbb3da96d80c707125f352b8c64ddac Mon Sep 17 00:00:00 2001 From: Felix Wang Date: Wed, 15 Sep 2021 16:39:05 -0700 Subject: [PATCH 6/6] Small fix Signed-off-by: Felix Wang --- sdk/python/feast/repo_config.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index 52853dba9c..b2a2d913ea 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -213,6 +213,10 @@ def _validate_feature_server_config(cls, values): if "feature_server" not in values: return values + # Skip if we aren't creating the configuration from a dict + if not isinstance(values["feature_server"], Dict): + return values + # Make sure that the provider configuration is set. We need it to set the defaults if "provider" not in values: raise FeastProviderNotSetError()