Merge branch 'master' into jacob_release_script

.github/workflows/integration_tests.yml

@@ -52,7 +52,7 @@ jobs:
       - name: Install dependencies
         run: make install-python-ci-dependencies
       - name: Test python
-        run: FEAST_USAGE=False pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests --integration
+        run: FEAST_USAGE=False IS_TEST=True pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests --integration
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v1
         with:

.github/workflows/pr_integration_tests.yml

@@ -63,7 +63,7 @@ jobs:
       - name: Install dependencies
         run: make install-python-ci-dependencies
       - name: Test python
-        run: FEAST_USAGE=False pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests --integration
+        run: FEAST_USAGE=False IS_TEST=True pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests --integration
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v1
         with:

Makefile

@@ -56,7 +56,7 @@ test-python:
 	FEAST_USAGE=False pytest -n 8 sdk/python/tests
 
 test-python-integration:
-	FEAST_USAGE=False pytest -n 8 --integration sdk/python/tests
+	FEAST_USAGE=False IS_TEST=True pytest -n 8 --integration sdk/python/tests
 
 format-python:
 	# Sort

OWNERS

@@ -3,9 +3,13 @@ approvers:
   - jklegar
   - tsotnet
   - achals
+  - adchia
+  - felixwang9817
 reviewers:
   - woop
   - jklegar
   - tsotnet
   - achals
   - tedhtchang
+  - adchia
+  - felixwang9817

README.md

@@ -22,7 +22,7 @@ Please see our [documentation](https://docs.feast.dev/) for more information abo
 ## Architecture
 <img src="https://i.imgur.com/IYUMF3Q.png" width="700">
 
-The above architecture is the minimal Feast deployment. Want to run the full Feast on Kubernetes? Click [here](https://docs.feast.dev/feast-on-kubernetes/getting-started).
+The above architecture is the minimal Feast deployment. Want to run the full Feast on GCP/AWS? Click [here](https://docs.feast.dev/how-to-guides/feast-gcp-aws).
 
 ## Getting Started
 
@@ -126,16 +126,17 @@ pprint(feature_vector)
 ## Important resources
 
 Please refer to the official documentation at [Documentation](https://docs.feast.dev/)
- * [Quickstart](https://docs.feast.dev/quickstart)
+ * [Quickstart](https://docs.feast.dev/getting-started/quickstart)
  * [Roadmap](https://docs.feast.dev/roadmap)
- * [Feast on Kubernetes](https://docs.feast.dev/feast-on-kubernetes/getting-started)
+ * [Tutorials](https://docs.feast.dev/tutorials/tutorials-overview)
+ * [Running Feast with GCP/AWS](https://docs.feast.dev/how-to-guides/feast-gcp-aws)
  * [Change Log](https://github.com/feast-dev/feast/blob/master/CHANGELOG.md)
  * [Slack (#Feast)](https://slack.feast.dev/)
 
 ## Contributing
 Feast is a community project and is still under active development. Please have a look at our contributing and development guides if you want to contribute to the project:
-- [Contribution Process for Feast](https://docs.feast.dev/contributing/contributing)
-- [Development Guide for Feast](https://docs.feast.dev/contributing/development-guide)
+- [Contribution Process for Feast](https://docs.feast.dev/project/contributing)
+- [Development Guide for Feast](https://docs.feast.dev/project/development-guide)
 - [Development Guide for the Main Feast Repository](./CONTRIBUTING.md)
 
 ## Contributors ✨

docs/README.md

@@ -49,8 +49,9 @@ The best way to learn Feast is to use it. Head over to our [Quickstart](getting-
 Explore the following resources to get started with Feast:
 
 * [Quickstart](getting-started/quickstart.md) is the fastest way to get started with Feast
-* [Getting started ](how-to-guides/feast-gcp-aws/)provides a step-by-step guide to using Feast.
-* [Concepts](getting-started/concepts/data-model-and-concepts/#concepts) describes all important Feast API concepts.
+* [Concepts](getting-started/concepts/) describes all important Feast API concepts and Feast's overall architecture.
+* [Tutorials](tutorials/tutorials-overview.md) shows full examples of using Feast in machine learning applications.
+* [Running Feast with GCP/AWS](how-to-guides/feast-gcp-aws/) provides a more in-depth guide to using Feast.
 * [Reference](reference/feast-cli-commands.md) contains detailed API and design documents.
 * [Contributing](project/contributing.md) contains resources for anyone who wants to contribute to Feast.
 

docs/reference/data-sources/file.md

@@ -4,6 +4,10 @@
 
 File data sources allow for the retrieval of historical feature values from files on disk for building training datasets, as well as for materializing features into an online store.
 
+{% hint style="warning" %}
+FileSource is meant for development purposes only and is not optimized for production use.
+{% endhint %}
+
 ## Example
 
 ```python

protos/feast/core/FeatureView.proto

@@ -35,6 +35,7 @@ message FeatureView {
     FeatureViewMeta meta = 2;
 }
 
+// TODO(adchia): refactor common fields from this and ODFV into separate metadata proto
 message FeatureViewSpec {
     // Name of the feature view. Must be unique. Not updated.
     string name = 1;

protos/feast/core/OnDemandFeatureView.proto

@@ -0,0 +1,57 @@
+//
+// Copyright 2020 The Feast Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+
+syntax = "proto3";
+package feast.core;
+
+option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/core";
+option java_outer_classname = "OnDemandFeatureViewProto";
+option java_package = "feast.proto.core";
+
+import "feast/core/FeatureView.proto";
+import "feast/core/Feature.proto";
+
+message OnDemandFeatureView {
+    // User-specified specifications of this feature view.
+    OnDemandFeatureViewSpec spec = 1;
+}
+
+message OnDemandFeatureViewSpec {
+    // Name of the feature view. Must be unique. Not updated.
+    string name = 1;
+
+    // Name of Feast project that this feature view belongs to.
+    string project = 2;
+
+    // List of features specifications for each feature defined with this feature view.
+    repeated FeatureSpecV2 features = 3;
+
+    // List of features specifications for each feature defined with this feature view.
+    // TODO(adchia): add support for request data
+    map<string, FeatureView> inputs = 4;
+
+    UserDefinedFunction user_defined_function = 5;
+}
+
+// Serialized representation of python function.
+message UserDefinedFunction {
+    // The function name
+    string name = 1;
+
+    // The python-syntax function body (serialized by dill)
+    bytes body = 2;
+}

protos/feast/core/Registry.proto

@@ -25,12 +25,14 @@ import "feast/core/Entity.proto";
 import "feast/core/FeatureService.proto";
 import "feast/core/FeatureTable.proto";
 import "feast/core/FeatureView.proto";
+import "feast/core/OnDemandFeatureView.proto";
 import "google/protobuf/timestamp.proto";
 
 message Registry {
     repeated Entity entities = 1;
     repeated FeatureTable feature_tables = 2;
     repeated FeatureView feature_views = 6;
+    repeated OnDemandFeatureView on_demand_feature_views = 8;
     repeated FeatureService feature_services = 7;
 
     string registry_schema_version = 3; // to support migrations; incremented when schema is changed

sdk/python/feast/__init__.py

@@ -13,6 +13,7 @@
 from .feature_store import FeatureStore
 from .feature_table import FeatureTable
 from .feature_view import FeatureView
+from .on_demand_feature_view import OnDemandFeatureView
 from .repo_config import RepoConfig
 from .value_type import ValueType
 
@@ -37,6 +38,7 @@
     "FeatureStore",
     "FeatureTable",
     "FeatureView",
+    "OnDemandFeatureView",
     "RepoConfig",
     "SourceType",
     "ValueType",

sdk/python/feast/cli.py

@@ -236,6 +236,58 @@ def feature_view_list(ctx: click.Context):
     print(tabulate(table, headers=["NAME", "ENTITIES"], tablefmt="plain"))
 
 
+@cli.group(name="on-demand-feature-views")
+def on_demand_feature_views_cmd():
+    """
+    Access feature views
+    """
+    pass
+
+
+@on_demand_feature_views_cmd.command("describe")
+@click.argument("name", type=click.STRING)
+@click.pass_context
+def on_demand_feature_view_describe(ctx: click.Context, name: str):
+    """
+    Describe an on demand feature view
+    """
+    repo = ctx.obj["CHDIR"]
+    cli_check_repo(repo)
+    store = FeatureStore(repo_path=str(repo))
+
+    try:
+        on_demand_feature_view = store.get_on_demand_feature_view(name)
+    except FeastObjectNotFoundException as e:
+        print(e)
+        exit(1)
+
+    print(
+        yaml.dump(
+            yaml.safe_load(str(on_demand_feature_view)),
+            default_flow_style=False,
+            sort_keys=False,
+        )
+    )
+
+
+@on_demand_feature_views_cmd.command(name="list")
+@click.pass_context
+def on_demand_feature_view_list(ctx: click.Context):
+    """
+    List all on demand feature views
+    """
+    repo = ctx.obj["CHDIR"]
+    cli_check_repo(repo)
+    store = FeatureStore(repo_path=str(repo))
+    table = []
+    for on_demand_feature_view in store.list_on_demand_feature_views():
+        table.append([on_demand_feature_view.name])
+
+    from tabulate import tabulate
+
+    print(tabulate(table, headers=["NAME"], tablefmt="plain"))
+
+
 @cli.command("apply", cls=NoOptionDefaultFormat)
 @click.option(
     "--skip-source-validation",

sdk/python/feast/errors.py

@@ -40,6 +40,16 @@ def __init__(self, name, project=None):
             super().__init__(f"Feature view {name} does not exist")
 
 
+class OnDemandFeatureViewNotFoundException(FeastObjectNotFoundException):
+    def __init__(self, name, project=None):
+        if project:
+            super().__init__(
+                f"On demand feature view {name} does not exist in project {project}"
+            )
+        else:
+            super().__init__(f"On demand feature view {name} does not exist")
+
+
 class FeatureTableNotFoundException(FeastObjectNotFoundException):
     def __init__(self, name, project=None):
         if project:
@@ -200,3 +210,10 @@ def __init__(self, entity_type: type):
             f"The entity dataframe you have provided must be a Pandas DataFrame or a SQL query, "
             f"but we found: {entity_type} "
         )
+
+
+class ConflictingFeatureViewNames(Exception):
+    def __init__(self, feature_view_name: str):
+        super().__init__(
+            f"The feature view name: {feature_view_name} refers to both an on-demand feature view and a feature view"
+        )