diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 0a25248d2a..62720a2363 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -167,10 +167,10 @@ jobs: steps: - uses: actions/checkout@v2 - name: Setup Node - uses: actions/setup-node@v2 - with: - node-version: '17.x' - registry-url: 'https://registry.npmjs.org' + uses: actions/setup-node@v2 + with: + node-version: '17.x' + registry-url: 'https://registry.npmjs.org' - name: Build wheels uses: pypa/cibuildwheel@v2.4.0 env: diff --git a/ui/feature_repo/features.py b/ui/feature_repo/features.py index 8a5e6153df..efdab694e8 100644 --- a/ui/feature_repo/features.py +++ b/ui/feature_repo/features.py @@ -1,27 +1,24 @@ from datetime import timedelta from feast import ( - Bool, Entity, FeatureService, FeatureView, Field, FileSource, - Int64, - String, ValueType, ) from feast.data_source import RequestSource from feast.request_feature_view import RequestFeatureView from feast.on_demand_feature_view import on_demand_feature_view -from feast.field import Field +from feast.types import Bool, Int64, String import pandas as pd zipcode = Entity( name="zipcode", value_type=ValueType.INT64, description="A zipcode", - labels={"owner": "danny@tecton.ai", "team": "hack week",}, + tags={"owner": "danny@tecton.ai", "team": "hack week",}, ) zipcode_source = FileSource( @@ -43,7 +40,7 @@ Field(name="population", dtype=Int64), Field(name="total_wages", dtype=Int64), ], - batch_source=zipcode_source, + source=zipcode_source, tags={ "date_added": "2022-02-7", "experiments": "experiment-A,experiment-B,experiment-C", @@ -64,7 +61,7 @@ Field(name="population", dtype=Int64), Field(name="total_wages", dtype=Int64), ], - batch_source=zipcode_source, + source=zipcode_source, tags={ "date_added": "2022-02-7", "experiments": "experiment-A,experiment-B,experiment-C", @@ -81,7 +78,7 @@ Field(name="tax_returns_filed", dtype=Int64), Field(name="total_wages", dtype=Int64), ], - batch_source=zipcode_source, + source=zipcode_source, tags={ "date_added": "2022-02-7", "experiments": "experiment-A,experiment-B,experiment-C", @@ -94,7 +91,7 @@ name="dob_ssn", value_type=ValueType.STRING, description="Date of birth and last four digits of social security number", - labels={"owner": "tony@tecton.ai", "team": "hack week",}, + tags={"owner": "tony@tecton.ai", "team": "hack week",}, ) credit_history_source = FileSource( @@ -119,7 +116,7 @@ Field(name="missed_payments_6m", dtype=Int64), Field(name="bankruptcies", dtype=Int64), ], - batch_source=credit_history_source, + source=credit_history_source, tags={ "date_added": "2022-02-6", "experiments": "experiment-A", @@ -152,16 +149,10 @@ def transaction_gt_last_credit_card_due(inputs: pd.DataFrame) -> pd.DataFrame: ) return df - -# Define request feature view -transaction_request_fv = RequestFeatureView( - name="transaction_request_fv", request_data_source=input_request, -) - model_v1 = FeatureService( name="credit_score_v1", features=[ - credit_history[["mortgage_due", "credit_card_due", "missed_payments_1y"]], + credit_history[["credit_card_due", "missed_payments_1y"]], zipcode_features, ], tags={"owner": "tony@tecton.ai", "stage": "staging"}, @@ -173,7 +164,6 @@ def transaction_gt_last_credit_card_due(inputs: pd.DataFrame) -> pd.DataFrame: features=[ credit_history[["mortgage_due", "credit_card_due", "missed_payments_1y"]], zipcode_features, - transaction_request_fv, ], tags={"owner": "tony@tecton.ai", "stage": "prod"}, description="Credit scoring model", diff --git a/ui/public/registry.json b/ui/public/registry.json index af328979e9..2d5c93c962 100644 --- a/ui/public/registry.json +++ b/ui/public/registry.json @@ -1,313 +1,104 @@ { - "project": "credit_scoring_aws", "dataSources": [ { - "type": "BATCH_FILE", - "eventTimestampColumn": "event_timestamp", "createdTimestampColumn": "created_timestamp", "fileOptions": { - "fileUrl": "data/credit_history.parquet" + "uri": "data/credit_history.parquet" }, "name": "credit_history", - "meta": { - "latestEventTimestamp": "2021-08-29T22:01:04.746575Z", - "earliestEventTimestamp": "2020-04-26T22:01:04.746575Z" - } + "timestampField": "event_timestamp", + "type": "BATCH_FILE" }, { - "type": "REQUEST_SOURCE", + "name": "transaction", "requestDataOptions": { - "schema": { - "transaction_amt": "INT64" - } + "schema": [ + { + "name": "transaction_amt", + "valueType": "INT64" + } + ] }, - "name": "transaction" + "type": "REQUEST_SOURCE" }, { - "type": "BATCH_FILE", - "eventTimestampColumn": "event_timestamp", "createdTimestampColumn": "created_timestamp", "fileOptions": { - "fileUrl": "data/zipcode_table.parquet" + "uri": "data/zipcode_table.parquet" }, "name": "zipcode", - "meta": { - "latestEventTimestamp": "2017-01-01T12:00:00Z", - "earliestEventTimestamp": "2017-01-01T12:00:00Z" - } + "timestampField": "event_timestamp", + "type": "BATCH_FILE" } ], "entities": [ { + "meta": { + "createdTimestamp": "2022-05-11T19:27:03.171062Z", + "lastUpdatedTimestamp": "2022-05-11T19:27:03.171062Z" + }, "spec": { + "joinKey": "__dummy_id", "name": "__dummy", - "valueType": "STRING", - "joinKey": "__dummy_id" - }, - "meta": { - "createdTimestamp": "2022-02-09T20:40:53.101387Z", - "lastUpdatedTimestamp": "2022-02-09T20:40:53.101387Z" + "valueType": "STRING" } }, { - "spec": { - "name": "dob_ssn", - "valueType": "STRING", - "description": "Date of birth and last four digits of social security number", - "joinKey": "dob_ssn", - "labels": { - "team": "hack week", - "owner": "tony@tecton.ai" - } - }, "meta": { - "createdTimestamp": "2022-02-09T20:40:53.101256Z", - "lastUpdatedTimestamp": "2022-02-09T20:40:53.101256Z" - } - }, - { - "spec": { - "name": "zipcode", - "valueType": "INT64", - "description": "A zipcode", - "joinKey": "zipcode", - "labels": { - "owner": "danny@tecton.ai", - "team": "hack week" - } + "createdTimestamp": "2022-05-11T19:27:03.171112Z", + "lastUpdatedTimestamp": "2022-05-11T19:27:03.171112Z" }, - "meta": { - "createdTimestamp": "2022-02-09T20:40:53.101335Z", - "lastUpdatedTimestamp": "2022-02-09T20:40:53.101335Z" - } - } - ], - "featureViews": [ - { "spec": { - "name": "credit_history", - "entities": ["dob_ssn"], - "features": [ - { - "name": "credit_card_due", - "valueType": "INT64" - }, - { - "name": "mortgage_due", - "valueType": "INT64" - }, - { - "name": "student_loan_due", - "valueType": "INT64" - }, - { - "name": "vehicle_loan_due", - "valueType": "INT64" - }, - { - "name": "hard_pulls", - "valueType": "INT64" - }, - { - "name": "missed_payments_2y", - "valueType": "INT64" - }, - { - "name": "missed_payments_1y", - "valueType": "INT64" - }, - { - "name": "missed_payments_6m", - "valueType": "INT64" - }, - { - "name": "bankruptcies", - "valueType": "INT64" - } - ], + "description": "Date of birth and last four digits of social security number", + "joinKey": "dob_ssn", + "name": "dob_ssn", "tags": { - "access_group": "feast-team@tecton.ai", - "experiments": "experiment-A", - "date_added": "2022-02-6" - }, - "ttl": "777600000s", - "batchSource": { - "type": "BATCH_FILE", - "eventTimestampColumn": "event_timestamp", - "createdTimestampColumn": "created_timestamp", - "fileOptions": { - "fileUrl": "data/credit_history.parquet" - }, - "dataSourceClassType": "feast.infra.offline_stores.file_source.FileSource", - "name": "credit_history", - "meta": { - "latestEventTimestamp": "2021-08-29T22:01:04.746575Z", - "earliestEventTimestamp": "2020-04-26T22:01:04.746575Z" - } + "owner": "tony@tecton.ai", + "team": "hack week" }, - "online": true - }, - "meta": { - "createdTimestamp": "2022-02-09T20:40:53.101460Z", - "lastUpdatedTimestamp": "2022-02-11T20:15:13.735432Z", - "materializationIntervals": [ - { - "startTime": "1997-06-20T20:41:14.456417Z", - "endTime": "2019-02-09T20:41:11Z" - }, - { - "startTime": "2019-02-09T20:41:11Z", - "endTime": "2022-02-09T20:42:03Z" - }, - { - "startTime": "2022-02-09T20:42:03Z", - "endTime": "2022-02-11T00:18:02Z" - } - ] + "valueType": "STRING" } }, { - "spec": { - "name": "zipcode_features", - "entities": ["zipcode"], - "features": [ - { - "name": "city", - "valueType": "STRING" - }, - { - "name": "state", - "valueType": "STRING" - }, - { - "name": "location_type", - "valueType": "STRING" - }, - { - "name": "tax_returns_filed", - "valueType": "INT64" - }, - { - "name": "population", - "valueType": "INT64" - }, - { - "name": "total_wages", - "valueType": "INT64" - } - ], - "tags": { - "experiments": "experiment-A,experiment-B,experiment-C", - "date_added": "2022-02-7", - "access_group": "feast-team@tecton.ai" - }, - "ttl": "315360000s", - "batchSource": { - "type": "BATCH_FILE", - "eventTimestampColumn": "event_timestamp", - "createdTimestampColumn": "created_timestamp", - "fileOptions": { - "fileUrl": "data/zipcode_table.parquet" - }, - "dataSourceClassType": "feast.infra.offline_stores.file_source.FileSource", - "name": "zipcode", - "meta": { - "latestEventTimestamp": "2017-01-01T12:00:00Z", - "earliestEventTimestamp": "2017-01-01T12:00:00Z" - } - }, - "online": true - }, "meta": { - "createdTimestamp": "2022-02-11T20:12:50.182923Z", - "lastUpdatedTimestamp": "2022-02-11T20:15:21.790447Z" - } - }, - { + "createdTimestamp": "2022-05-11T19:27:03.171153Z", + "lastUpdatedTimestamp": "2022-05-11T19:27:03.171153Z" + }, "spec": { - "name": "zipcode_money_features", - "entities": ["zipcode"], - "features": [ - { - "name": "tax_returns_filed", - "valueType": "INT64" - }, - { - "name": "total_wages", - "valueType": "INT64" - } - ], + "description": "A zipcode", + "joinKey": "zipcode", + "name": "zipcode", "tags": { - "experiments": "experiment-A,experiment-B,experiment-C", - "access_group": "feast-team@tecton.ai", - "date_added": "2022-02-7", - "test_apple": "2022-02-7", - "test_banana": "2022-02-7", - "test_cherry": "2022-02-7", - "test_danish": "2022-02-7", - "test_eggplant": "2022-02-7", - "test_figs": "2022-02-7", - "test_grape": "2022-02-7", - "test_honey": "2022-02-7", - "test_ice": "2022-02-7", - "test_jackfruit": "2022-02-7", - "test_kiwi_fruit": "2022-02-7", - "test_lychee": "2022-02-7", - "test_mango": "2022-02-7", - "test_orange": "2022-02-7", - "test_peach": "2022-02-7", - "test_question": "2022-02-7", - "test_ruby": "2022-02-7", - "test_starfruit": "2022-02-7", - "test_tamarind": "2022-02-7" - }, - "ttl": "315360000s", - "batchSource": { - "type": "BATCH_FILE", - "eventTimestampColumn": "event_timestamp", - "createdTimestampColumn": "created_timestamp", - "fileOptions": { - "fileUrl": "data/zipcode_table.parquet" - }, - "dataSourceClassType": "feast.infra.offline_stores.file_source.FileSource", - "name": "zipcode", - "meta": { - "latestEventTimestamp": "2017-01-01T12:00:00Z", - "earliestEventTimestamp": "2017-01-01T12:00:00Z" - } + "owner": "danny@tecton.ai", + "team": "hack week" }, - "online": true - }, - "meta": { - "createdTimestamp": "2022-02-11T20:10:53.228047Z", - "lastUpdatedTimestamp": "2022-02-11T20:15:15.949101Z" + "valueType": "INT64" } } ], "featureServices": [ { + "meta": { + "createdTimestamp": "2022-05-11T19:27:03.172623Z", + "lastUpdatedTimestamp": "2022-05-11T19:27:03.172623Z" + }, "spec": { - "name": "credit_score_v1", + "description": "Credit scoring model", "features": [ { - "featureViewName": "credit_history", "featureColumns": [ { "name": "credit_card_due", "valueType": "INT64" }, - { - "name": "mortgage_due", - "valueType": "INT64" - }, { "name": "missed_payments_1y", "valueType": "INT64" } - ] + ], + "featureViewName": "credit_history" }, { - "featureViewName": "zipcode_features", "featureColumns": [ { "name": "city", @@ -333,25 +124,26 @@ "name": "total_wages", "valueType": "INT64" } - ] + ], + "featureViewName": "zipcode_features" } ], + "name": "credit_score_v1", "tags": { "owner": "tony@tecton.ai", "stage": "staging" - }, - "description": "Credit scoring model" - }, - "meta": { - "createdTimestamp": "2022-02-11T20:12:50.186773Z" + } } }, { + "meta": { + "createdTimestamp": "2022-05-11T19:27:03.172405Z", + "lastUpdatedTimestamp": "2022-05-11T19:27:03.172405Z" + }, "spec": { - "name": "credit_score_v2", + "description": "Credit scoring model", "features": [ { - "featureViewName": "credit_history", "featureColumns": [ { "name": "credit_card_due", @@ -365,10 +157,10 @@ "name": "missed_payments_1y", "valueType": "INT64" } - ] + ], + "featureViewName": "credit_history" }, { - "featureViewName": "zipcode_features", "featureColumns": [ { "name": "city", @@ -394,34 +186,26 @@ "name": "total_wages", "valueType": "INT64" } - ] - }, - { - "featureViewName": "transaction_request_fv", - "featureColumns": [ - { - "name": "transaction_amt", - "valueType": "INT64" - } - ] + ], + "featureViewName": "zipcode_features" } ], + "name": "credit_score_v2", "tags": { - "stage": "prod", - "owner": "tony@tecton.ai" - }, - "description": "Credit scoring model" - }, - "meta": { - "createdTimestamp": "2022-02-11T20:12:50.185785Z" + "owner": "tony@tecton.ai", + "stage": "prod" + } } }, { + "meta": { + "createdTimestamp": "2022-05-11T19:27:03.172264Z", + "lastUpdatedTimestamp": "2022-05-11T19:27:03.172264Z" + }, "spec": { - "name": "credit_score_v3", + "description": "Credit scoring model", "features": [ { - "featureViewName": "credit_history", "featureColumns": [ { "name": "credit_card_due", @@ -435,10 +219,10 @@ "name": "missed_payments_1y", "valueType": "INT64" } - ] + ], + "featureViewName": "credit_history" }, { - "featureViewName": "zipcode_features", "featureColumns": [ { "name": "city", @@ -464,34 +248,35 @@ "name": "total_wages", "valueType": "INT64" } - ] + ], + "featureViewName": "zipcode_features" }, { - "featureViewName": "transaction_gt_last_credit_card_due", "featureColumns": [ { "name": "transaction_gt_last_credit_card_due", "valueType": "BOOL" } - ] + ], + "featureViewName": "transaction_gt_last_credit_card_due" } ], + "name": "credit_score_v3", "tags": { - "stage": "dev", - "owner": "tony@tecton.ai" - }, - "description": "Credit scoring model" - }, - "meta": { - "createdTimestamp": "2022-02-11T20:12:50.186367Z" + "owner": "tony@tecton.ai", + "stage": "dev" + } } }, { + "meta": { + "createdTimestamp": "2022-05-11T19:27:03.172530Z", + "lastUpdatedTimestamp": "2022-05-11T19:27:03.172530Z" + }, "spec": { - "name": "zipcode_model", + "description": "Location model", "features": [ { - "featureViewName": "zipcode_features", "featureColumns": [ { "name": "city", @@ -517,25 +302,26 @@ "name": "total_wages", "valueType": "INT64" } - ] + ], + "featureViewName": "zipcode_features" } ], + "name": "zipcode_model", "tags": { - "stage": "dev", - "owner": "amanda@tecton.ai" - }, - "description": "Location model" - }, - "meta": { - "createdTimestamp": "2022-02-11T20:12:50.187069Z" + "owner": "amanda@tecton.ai", + "stage": "dev" + } } }, { + "meta": { + "createdTimestamp": "2022-05-11T19:27:03.172188Z", + "lastUpdatedTimestamp": "2022-05-11T19:27:03.172188Z" + }, "spec": { - "name": "zipcode_model_v2", + "description": "Location model", "features": [ { - "featureViewName": "zipcode_money_features", "featureColumns": [ { "name": "tax_returns_filed", @@ -545,45 +331,242 @@ "name": "total_wages", "valueType": "INT64" } - ] + ], + "featureViewName": "zipcode_money_features" } ], + "name": "zipcode_model_v2", "tags": { "owner": "amanda@tecton.ai", "stage": "dev" + } + } + } + ], + "featureViews": [ + { + "meta": { + "createdTimestamp": "2022-05-11T19:27:03.171421Z", + "lastUpdatedTimestamp": "2022-05-11T19:28:21.444739Z", + "materializationIntervals": [ + { + "endTime": "2019-05-11T19:27:05Z", + "startTime": "1997-09-19T19:27:13.273753Z" + }, + { + "endTime": "2022-05-11T19:27:43Z", + "startTime": "2019-05-11T19:27:05Z" + } + ] + }, + "spec": { + "batchSource": { + "createdTimestampColumn": "created_timestamp", + "dataSourceClassType": "feast.infra.offline_stores.file_source.FileSource", + "fileOptions": { + "uri": "data/credit_history.parquet" + }, + "name": "credit_history", + "timestampField": "event_timestamp", + "type": "BATCH_FILE" + }, + "entities": [ + "dob_ssn" + ], + "features": [ + { + "name": "credit_card_due", + "valueType": "INT64" + }, + { + "name": "mortgage_due", + "valueType": "INT64" + }, + { + "name": "student_loan_due", + "valueType": "INT64" + }, + { + "name": "vehicle_loan_due", + "valueType": "INT64" + }, + { + "name": "hard_pulls", + "valueType": "INT64" + }, + { + "name": "missed_payments_2y", + "valueType": "INT64" + }, + { + "name": "missed_payments_1y", + "valueType": "INT64" + }, + { + "name": "missed_payments_6m", + "valueType": "INT64" + }, + { + "name": "bankruptcies", + "valueType": "INT64" + } + ], + "name": "credit_history", + "online": true, + "tags": { + "access_group": "feast-team@tecton.ai", + "date_added": "2022-02-6", + "experiments": "experiment-A" }, - "description": "Location model" + "ttl": "777600000s" + } + }, + { + "meta": { + "createdTimestamp": "2022-05-11T19:27:03.171300Z", + "lastUpdatedTimestamp": "2022-05-11T19:27:46.002348Z", + "materializationIntervals": [ + { + "endTime": "2019-05-11T19:27:05Z", + "startTime": "2012-05-13T19:27:08.483036Z" + }, + { + "endTime": "2022-05-11T19:27:43Z", + "startTime": "2019-05-11T19:27:05Z" + } + ] }, + "spec": { + "batchSource": { + "createdTimestampColumn": "created_timestamp", + "dataSourceClassType": "feast.infra.offline_stores.file_source.FileSource", + "fileOptions": { + "uri": "data/zipcode_table.parquet" + }, + "name": "zipcode", + "timestampField": "event_timestamp", + "type": "BATCH_FILE" + }, + "entities": [ + "zipcode" + ], + "features": [ + { + "name": "city", + "valueType": "STRING" + }, + { + "name": "state", + "valueType": "STRING" + }, + { + "name": "location_type", + "valueType": "STRING" + }, + { + "name": "tax_returns_filed", + "valueType": "INT64" + }, + { + "name": "population", + "valueType": "INT64" + }, + { + "name": "total_wages", + "valueType": "INT64" + } + ], + "name": "zipcode_features", + "online": true, + "tags": { + "access_group": "feast-team@tecton.ai", + "date_added": "2022-02-7", + "experiments": "experiment-A,experiment-B,experiment-C" + }, + "ttl": "315360000s" + } + }, + { "meta": { - "createdTimestamp": "2022-02-11T20:17:15.582561Z" + "createdTimestamp": "2022-05-11T19:27:03.171195Z", + "lastUpdatedTimestamp": "2022-05-11T19:27:45.942549Z", + "materializationIntervals": [ + { + "endTime": "2019-05-11T19:27:05Z", + "startTime": "2012-05-13T19:27:06.493847Z" + }, + { + "endTime": "2022-05-11T19:27:43Z", + "startTime": "2019-05-11T19:27:05Z" + } + ] + }, + "spec": { + "batchSource": { + "createdTimestampColumn": "created_timestamp", + "dataSourceClassType": "feast.infra.offline_stores.file_source.FileSource", + "fileOptions": { + "uri": "data/zipcode_table.parquet" + }, + "name": "zipcode", + "timestampField": "event_timestamp", + "type": "BATCH_FILE" + }, + "entities": [ + "zipcode" + ], + "features": [ + { + "name": "tax_returns_filed", + "valueType": "INT64" + }, + { + "name": "total_wages", + "valueType": "INT64" + } + ], + "name": "zipcode_money_features", + "online": true, + "tags": { + "access_group": "feast-team@tecton.ai", + "date_added": "2022-02-7", + "experiments": "experiment-A,experiment-B,experiment-C" + }, + "ttl": "315360000s" } } ], + "infra": [ + { + "name": "credit_scoring_aws_credit_history", + "path": "/Users/dannychiao/GitHub/feast/ui/feature_repo/data/online.db" + }, + { + "name": "credit_scoring_aws_zipcode_features", + "path": "/Users/dannychiao/GitHub/feast/ui/feature_repo/data/online.db" + }, + { + "name": "credit_scoring_aws_zipcode_money_features", + "path": "/Users/dannychiao/GitHub/feast/ui/feature_repo/data/online.db" + } + ], "onDemandFeatureViews": [ { + "meta": { + "createdTimestamp": "2022-05-11T19:27:03.171556Z", + "lastUpdatedTimestamp": "2022-05-11T19:27:03.171556Z" + }, "spec": { - "name": "transaction_gt_last_credit_card_due", "features": [ { "name": "transaction_gt_last_credit_card_due", "valueType": "BOOL" } ], - "inputs": { - "transaction": { - "requestDataSource": { - "type": "REQUEST_SOURCE", - "requestDataOptions": { - "schema": { - "transaction_amt": "INT64" - } - }, - "name": "transaction" - } - }, + "name": "transaction_gt_last_credit_card_due", + "sources": { "credit_history": { "featureViewProjection": { - "featureViewName": "credit_history", "featureColumns": [ { "name": "credit_card_due", @@ -621,169 +604,31 @@ "name": "bankruptcies", "valueType": "INT64" } - ] - } - } - }, - "userDefinedFunction": { - "name": "transaction_gt_last_credit_card_due", - "body": "@on_demand_feature_view(\n sources={\"credit_history\": credit_history, \"transaction\": input_request,},\n schema=[\n Field(name=\"transaction_gt_last_credit_card_due\", dtype=Bool),\n ],\n)\ndef transaction_gt_last_credit_card_due(inputs: pd.DataFrame) -> pd.DataFrame:\n df = pd.DataFrame()\n df[\"transaction_gt_last_credit_card_due\"] = (\n inputs[\"transaction_amt\"] > inputs[\"credit_card_due\"]\n )\n return df\n" - } - }, - "meta": { - "createdTimestamp": "2022-02-11T20:17:15.581514Z", - "lastUpdatedTimestamp": "2022-02-11T20:17:15.581514Z" - } - } - ], - "requestFeatureViews": [ - { - "spec": { - "name": "transaction_request_fv", - "requestDataSource": { - "type": "REQUEST_SOURCE", - "requestDataOptions": { - "schema": { - "transaction_amt": "INT64" + ], + "featureViewName": "credit_history" } }, - "name": "transaction" - } - } - } - ], - "savedDatasets": [ - { - "spec": { - "name": "my_training_ds", - "features": [ - "credit_history:credit_card_due", - "credit_history:mortgage_due", - "credit_history:missed_payments_1y", - "zipcode_features:city", - "zipcode_features:state", - "zipcode_features:location_type", - "zipcode_features:tax_returns_filed", - "zipcode_features:population", - "zipcode_features:total_wages" - ], - "joinKeys": [ - "person_income", - "person_emp_length", - "created_timestamp", - "zipcode", - "person_home_ownership", - "loan_amnt", - "person_age", - "loan_int_rate", - "loan_status", - "loan_id", - "dob_ssn", - "loan_intent" - ], - "storage": { - "fileStorage": { - "fileFormat": { - "parquetFormat": {} - }, - "fileUrl": "my_training_ds.parquet" - } - }, - "featureService": { - "spec": { - "name": "credit_score_v1", - "features": [ - { - "featureViewName": "credit_history", - "featureColumns": [ - { - "name": "credit_card_due", - "valueType": "INT64" - }, - { - "name": "mortgage_due", - "valueType": "INT64" - }, + "transaction": { + "requestDataSource": { + "name": "transaction", + "requestDataOptions": { + "schema": [ { - "name": "missed_payments_1y", + "name": "transaction_amt", "valueType": "INT64" } ] }, - { - "featureViewName": "zipcode_features", - "featureColumns": [ - { - "name": "city", - "valueType": "STRING" - }, - { - "name": "state", - "valueType": "STRING" - }, - { - "name": "location_type", - "valueType": "STRING" - }, - { - "name": "tax_returns_filed", - "valueType": "INT64" - }, - { - "name": "population", - "valueType": "INT64" - }, - { - "name": "total_wages", - "valueType": "INT64" - } - ] - } - ], - "tags": { - "stage": "staging", - "owner": "tony@tecton.ai" - }, - "description": "Credit scoring model" - }, - "meta": { - "createdTimestamp": "2022-02-09T20:40:53.103078Z" + "type": "REQUEST_SOURCE" + } } }, - "profile": "{\"meta\": {\"great_expectations_version\": \"0.14.4\"}, \"expectations\": [{\"meta\": {}, \"kwargs\": {\"column\": \"credit_card_due\", \"min_value\": 0, \"mostly\": 0.99}, \"expectation_type\": \"expect_column_values_to_be_between\"}, {\"meta\": {}, \"kwargs\": {\"column\": \"missed_payments_1y\", \"min_value\": 0, \"max_value\": 5, \"mostly\": 0.99}, \"expectation_type\": \"expect_column_values_to_be_between\"}], \"data_asset_type\": \"Dataset\", \"expectation_suite_name\": \"default\", \"ge_cloud_id\": null}" - }, - "meta": { - "createdTimestamp": "2022-02-09T20:44:03.377806Z", - "minEventTimestamp": "2020-08-25T20:34:41.361Z", - "maxEventTimestamp": "2021-08-25T20:34:41.361Z" + "userDefinedFunction": { + "body": "@on_demand_feature_view(\n sources=[credit_history, input_request],\n schema=[\n Field(name=\"transaction_gt_last_credit_card_due\", dtype=Bool),\n ],\n)\ndef transaction_gt_last_credit_card_due(inputs: pd.DataFrame) -> pd.DataFrame:\n df = pd.DataFrame()\n df[\"transaction_gt_last_credit_card_due\"] = (\n inputs[\"transaction_amt\"] > inputs[\"credit_card_due\"]\n )\n return df\n", + "name": "transaction_gt_last_credit_card_due" + } } } ], - "infra": [ - { - "infraObjects": [ - { - "infraObjectClassType": "feast.infra.online_stores.sqlite.SqliteTable", - "sqliteTable": { - "path": "/Users/dannychiao/GitHub/feast-ui/feature_repo/data/online.db", - "name": "credit_scoring_aws_credit_history" - } - }, - { - "infraObjectClassType": "feast.infra.online_stores.sqlite.SqliteTable", - "sqliteTable": { - "path": "/Users/dannychiao/GitHub/feast-ui/feature_repo/data/online.db", - "name": "credit_scoring_aws_zipcode_features" - } - }, - { - "infraObjectClassType": "feast.infra.online_stores.sqlite.SqliteTable", - "sqliteTable": { - "path": "/Users/dannychiao/GitHub/feast-ui/feature_repo/data/online.db", - "name": "credit_scoring_aws_zipcode_money_features" - } - } - ] - } - ] + "project": "credit_scoring_aws" } diff --git a/ui/src/components/TagsDisplay.tsx b/ui/src/components/TagsDisplay.tsx index 6f7f23b007..87e4863847 100644 --- a/ui/src/components/TagsDisplay.tsx +++ b/ui/src/components/TagsDisplay.tsx @@ -9,11 +9,36 @@ import EuiCustomLink from "./EuiCustomLink"; interface TagsDisplayProps { createLink?: (key: string, value: string) => string; tags: Record; + owner?: string; + description?: string; } -const TagsDisplay = ({ tags, createLink }: TagsDisplayProps) => { +const TagsDisplay = ({ + tags, + createLink, + owner, + description, +}: TagsDisplayProps) => { return ( + {owner ? ( + + owner + {owner} + + ) : ( + "" + )} + {description ? ( + + description + + {description} + + + ) : ( + "" + )} {Object.entries(tags).map(([key, value]) => { return ( diff --git a/ui/src/index.tsx b/ui/src/index.tsx index d4f1013503..df12722a64 100644 --- a/ui/src/index.tsx +++ b/ui/src/index.tsx @@ -75,8 +75,15 @@ ReactDOM.render( reactQueryClient={queryClient} feastUIConfigs={{ tabsRegistry: tabsRegistry, + projectListPromise: fetch("http://0.0.0.0:8888/projects-list", { + headers: { + "Content-Type": "application/json", + }, + }).then((res) => { + return res.json(); + }) }} /> , document.getElementById("root") -); +); \ No newline at end of file diff --git a/ui/src/pages/data-sources/BatchSourcePropertiesView.tsx b/ui/src/pages/data-sources/BatchSourcePropertiesView.tsx index 3bc7276203..97f9f58d27 100644 --- a/ui/src/pages/data-sources/BatchSourcePropertiesView.tsx +++ b/ui/src/pages/data-sources/BatchSourcePropertiesView.tsx @@ -1,115 +1,136 @@ import React from "react"; import { - EuiCodeBlock, - EuiDescriptionList, - EuiDescriptionListDescription, - EuiDescriptionListTitle, - EuiFlexGroup, - EuiFlexItem, - EuiSpacer, - EuiTitle, - } from "@elastic/eui"; + EuiCodeBlock, + EuiDescriptionList, + EuiDescriptionListDescription, + EuiDescriptionListTitle, + EuiFlexGroup, + EuiFlexItem, + EuiSpacer, + EuiTitle, +} from "@elastic/eui"; interface BatchSourcePropertiesViewProps { - batchSource: { - type?: string | undefined; - dataSourceClassType?: string | undefined; - fileOptions?: { - fileUrl?: string | undefined; - } | undefined; - meta?: { - latestEventTimestamp?: Date | undefined; - earliestEventTimestamp?: Date | undefined; - } | undefined; - bigqueryOptions?: { - dbtModelSerialized?: string | undefined - } | undefined; - } + batchSource: { + type?: string | undefined; + owner?: string | undefined; + description?: string | undefined; + dataSourceClassType?: string | undefined; + fileOptions?: + | { + uri?: string | undefined; + } + | undefined; + meta?: + | { + latestEventTimestamp?: Date | undefined; + earliestEventTimestamp?: Date | undefined; + } + | undefined; + bigqueryOptions?: + | { + dbtModelSerialized?: string | undefined; + } + | undefined; + }; } const BatchSourcePropertiesView = (props: BatchSourcePropertiesViewProps) => { - const batchSource = props.batchSource; - return ( - - - - - {(batchSource.dataSourceClassType || batchSource.type) && - - Source Type - {batchSource.dataSourceClassType ? ( - - {batchSource.dataSourceClassType - .split(".") - .at(-1)} - ) - : batchSource.type ? ( - {batchSource.type} - ) - : ""} - } - - {batchSource.fileOptions && ( - - - File URL - - - {batchSource.fileOptions - ? batchSource.fileOptions.fileUrl - : ""} - - - )} - {batchSource.meta?.latestEventTimestamp && ( - - - Latest Event - - - {batchSource.meta.latestEventTimestamp.toLocaleDateString( - "en-CA" - )} - - - )} - {batchSource.meta?.earliestEventTimestamp && ( - - - Earliest Event - - - {batchSource.meta.earliestEventTimestamp.toLocaleDateString( - "en-CA" - )} - - - )} - - - - {batchSource.bigqueryOptions?.dbtModelSerialized && ( - - - - )} - {batchSource.bigqueryOptions?.dbtModelSerialized && ( - - -

Dbt Transformation

-
- - {batchSource.bigqueryOptions.dbtModelSerialized} - -
+ const batchSource = props.batchSource; + return ( + + + + + {(batchSource.dataSourceClassType || batchSource.type) && ( + + Source Type + {batchSource.dataSourceClassType ? ( + + {batchSource.dataSourceClassType.split(".").at(-1)} + + ) : batchSource.type ? ( + + {batchSource.type} + + ) : ( + "" )} - - ); +
+ )} + + {batchSource.owner && ( + + Owner + + {batchSource.owner} + + + )} + {batchSource.description && ( + + Description + + {batchSource.description} + + + )} + {batchSource.fileOptions && ( + + File URL + + {batchSource.fileOptions ? batchSource.fileOptions.uri : ""} + + + )} + {batchSource.meta?.latestEventTimestamp && ( + + Latest Event + + {batchSource.meta.latestEventTimestamp.toLocaleDateString( + "en-CA" + )} + + + )} + {batchSource.meta?.earliestEventTimestamp && ( + + + Earliest Event + + + {batchSource.meta.earliestEventTimestamp.toLocaleDateString( + "en-CA" + )} + + + )} +
+ + + {batchSource.bigqueryOptions?.dbtModelSerialized && ( + + + + )} + {batchSource.bigqueryOptions?.dbtModelSerialized && ( + + +

Dbt Transformation

+
+ + {batchSource.bigqueryOptions.dbtModelSerialized} + +
+ )} + + + ); }; export default BatchSourcePropertiesView; diff --git a/ui/src/pages/data-sources/DataSourceOverviewTab.tsx b/ui/src/pages/data-sources/DataSourceOverviewTab.tsx index f7c05000e7..124a0e6ab9 100644 --- a/ui/src/pages/data-sources/DataSourceOverviewTab.tsx +++ b/ui/src/pages/data-sources/DataSourceOverviewTab.tsx @@ -27,7 +27,6 @@ const DataSourceOverviewTab = () => { const { isLoading, isSuccess, isError, data, consumingFeatureViews } = useLoadDataSource(dsName); const isEmpty = data === undefined; - console.log(consumingFeatureViews); return ( @@ -51,7 +50,7 @@ const DataSourceOverviewTab = () => { {data.fileOptions || data.bigqueryOptions ? ( - ) : data.requestDataOptions ? ( + ) : data.type ? ( @@ -62,7 +61,7 @@ const DataSourceOverviewTab = () => { - ): ( + ) : ( "" )} @@ -78,12 +77,10 @@ const DataSourceOverviewTab = () => { { + fields={data.requestDataOptions.schema.map((obj) => { return { - fieldName: field, - valueType: type, + fieldName: obj.name, + valueType: obj.valueType, }; })} /> diff --git a/ui/src/pages/feature-views/OnDemandFeatureViewOverviewTab.tsx b/ui/src/pages/feature-views/OnDemandFeatureViewOverviewTab.tsx index 9bd725534e..1ea509d8df 100644 --- a/ui/src/pages/feature-views/OnDemandFeatureViewOverviewTab.tsx +++ b/ui/src/pages/feature-views/OnDemandFeatureViewOverviewTab.tsx @@ -38,7 +38,7 @@ const whereFSconsumesThisFv = (fvName: string) => { const OnDemandFeatureViewOverviewTab = ({ data, }: OnDemandFeatureViewOverviewTabProps) => { - const inputs = Object.entries(data.spec.inputs); + const inputs = Object.entries(data.spec.sources); const relationshipQuery = useLoadRelationshipData(); const fsNames = relationshipQuery.data diff --git a/ui/src/pages/feature-views/RegularFeatureViewOverviewTab.tsx b/ui/src/pages/feature-views/RegularFeatureViewOverviewTab.tsx index 72ea646c95..d284d697e8 100644 --- a/ui/src/pages/feature-views/RegularFeatureViewOverviewTab.tsx +++ b/ui/src/pages/feature-views/RegularFeatureViewOverviewTab.tsx @@ -60,27 +60,6 @@ const RegularFeatureViewOverviewTab = ({ - {data.spec.batchSource.meta ? ( - - - - ) : ( - No batchSource specified on this feature view. - )} - {data.meta.lastUpdatedTimestamp && ( - - - - )} @@ -96,7 +75,7 @@ const RegularFeatureViewOverviewTab = ({ features={data.spec.features} /> ) : ( - No Tags specified on this feature view. + No features specified on this feature view. )} @@ -156,6 +135,8 @@ const RegularFeatureViewOverviewTab = ({ encodeSearchQueryString(`${key}:${value}`) ); }} + owner={data.spec.owner} + description={data.spec.description} /> ) : ( No Tags specified on this feature view. diff --git a/ui/src/parsers/feastDatasources.ts b/ui/src/parsers/feastDatasources.ts index a9f58d716c..3e1dca72d1 100644 --- a/ui/src/parsers/feastDatasources.ts +++ b/ui/src/parsers/feastDatasources.ts @@ -1,20 +1,22 @@ import { z } from "zod"; -import { FEAST_FEATURE_VALUE_TYPES } from "./types"; +import { FeastFeatureColumnSchema } from "./feastFeatureViews"; const FeastDatasourceSchema = z.object({ type: z.string(), eventTimestampColumn: z.string().optional(), createdTimestampColumn: z.string().optional(), fileOptions: z.object({ - fileUrl: z.string().optional(), + uri: z.string().optional(), }).optional(), name: z.string(), + description: z.string().optional(), + owner: z.string().optional(), meta: z.object({ latestEventTimestamp: z.string().transform((val) => new Date(val)), earliestEventTimestamp: z.string().transform((val) => new Date(val)), }).optional(), requestDataOptions: z.object({ - schema: z.record(z.nativeEnum(FEAST_FEATURE_VALUE_TYPES)), + schema: z.array(FeastFeatureColumnSchema), }).optional(), bigqueryOptions: z.object({ tableRef: z.string().optional(), diff --git a/ui/src/parsers/feastFeatureViews.ts b/ui/src/parsers/feastFeatureViews.ts index c8cdadd25c..3e63b5afd0 100644 --- a/ui/src/parsers/feastFeatureViews.ts +++ b/ui/src/parsers/feastFeatureViews.ts @@ -4,6 +4,7 @@ import { FEAST_FEATURE_VALUE_TYPES } from "./types"; const FeastFeatureColumnSchema = z.object({ name: z.string(), valueType: z.nativeEnum(FEAST_FEATURE_VALUE_TYPES), + tags: z.record(z.string()).optional(), }); const FeastBatchSourceSchema = z.object({ @@ -11,9 +12,11 @@ const FeastBatchSourceSchema = z.object({ eventTimestampColumn: z.string().optional(), createdTimestampColumn: z.string().optional(), fileOptions: z.object({ - fileUrl: z.string().optional(), + uri: z.string().optional(), }).optional(), name: z.string().optional(), + description: z.string().optional(), + owner: z.string().optional(), meta: z.object({ earliestEventTimestamp: z.string().transform((val) => new Date(val)), latestEventTimestamp: z.string().transform((val) => new Date(val)), @@ -30,12 +33,14 @@ const FeastBatchSourceSchema = z.object({ const FeastFeatureViewSchema = z.object({ spec: z.object({ + description: z.string().optional(), name: z.string(), entities: z.array(z.string()), features: z.array(FeastFeatureColumnSchema), ttl: z.string().transform((val) => parseInt(val)), batchSource: FeastBatchSourceSchema, online: z.boolean(), + owner: z.string().optional(), tags: z.record(z.string()).optional(), }), meta: z.object({ diff --git a/ui/src/parsers/feastODFVS.ts b/ui/src/parsers/feastODFVS.ts index ebac09e163..8341438d50 100644 --- a/ui/src/parsers/feastODFVS.ts +++ b/ui/src/parsers/feastODFVS.ts @@ -14,7 +14,7 @@ const RequestDataSourceSchema = z.object({ type: z.string(), name: z.string(), requestDataOptions: z.object({ - schema: z.record(z.nativeEnum(FEAST_FEATURE_VALUE_TYPES)), + schema: z.array(FeastFeatureColumnSchema), }), }), }); @@ -28,7 +28,7 @@ const FeastODFVSchema = z.object({ spec: z.object({ name: z.string(), features: z.array(FeastFeatureColumnSchema), - inputs: z.record(ODFVInputsSchema), + sources: z.record(ODFVInputsSchema), userDefinedFunction: z.object({ name: z.string(), body: z.string(), diff --git a/ui/src/parsers/parseEntityRelationships.ts b/ui/src/parsers/parseEntityRelationships.ts index bf82e86ff9..f54bff63a1 100644 --- a/ui/src/parsers/parseEntityRelationships.ts +++ b/ui/src/parsers/parseEntityRelationships.ts @@ -57,7 +57,7 @@ const parseEntityRelationships = (objects: FeastRegistryType) => { }); objects.onDemandFeatureViews?.forEach((fv) => { - Object.values(fv.spec.inputs).forEach((input: { [key: string]: any }) => { + Object.values(fv.spec.sources).forEach((input: { [key: string]: any }) => { if (input.requestDataSource) { links.push({ source: {