Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: Substrait ODFVs for online #4064

Merged
merged 4 commits into from
Apr 2, 2024
Merged

fix: Substrait ODFVs for online #4064

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
47e0e37
fix substrait odfvs for online, add tests
tokoko Apr 2, 2024
9de3fde
fix formatting
tokoko Apr 2, 2024
7b33fa2
change odfv substrait test dates relative to start_date and end_date
tokoko Apr 2, 2024
1bf360a
force tests rerun
tokoko Apr 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 4 additions & 5 deletions sdk/python/feast/feature_store.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2037,11 +2037,10 @@ def _augment_response_with_on_demand_transforms(

proto_values = []
for selected_feature in selected_subset:
if odfv.mode in ["python", "pandas"]:
feature_vector = transformed_features[selected_feature]
proto_values.append(
python_values_to_proto_values(feature_vector, ValueType.UNKNOWN)
)
feature_vector = transformed_features[selected_feature]
proto_values.append(
python_values_to_proto_values(feature_vector, ValueType.UNKNOWN)
)

odfv_result_names |= set(selected_subset)

Expand Down
2 changes: 1 addition & 1 deletion sdk/python/feast/infra/offline_stores/offline_store.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ def to_arrow(
features_df = self._to_df_internal(timeout=timeout)
if self.on_demand_feature_views:
for odfv in self.on_demand_feature_views:
if odfv.mode != "pandas":
if odfv.mode not in {"pandas", "substrait"}:
raise Exception(
f'OnDemandFeatureView mode "{odfv.mode}" not supported for offline processing.'
)
Expand Down
4 changes: 3 additions & 1 deletion sdk/python/feast/on_demand_feature_view.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -465,7 +465,9 @@ def get_transformed_features(
return self.get_transformed_features_dict(
feature_dict=features,
)
elif self.mode == "pandas" and isinstance(features, pd.DataFrame):
elif self.mode in {"pandas", "substrait"} and isinstance(
features, pd.DataFrame
):
return self.get_transformed_features_df(
df_with_features=features,
full_feature_names=full_feature_names,
Expand Down
42 changes: 31 additions & 11 deletions ...est_on_demand_substrait_transformation.py → ...sts/unit/test_substrait_transformation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,11 @@ def substrait_view(inputs: Table) -> Table:
[driver, driver_stats_source, driver_stats_fv, substrait_view, pandas_view]
)

store.materialize(
start_date=datetime(2000, 4, 12, 10, 59, 42),
end_date=datetime(3000, 4, 12, 10, 59, 42),
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

probably better to do make these relate dates using datetime.now()

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

😄 actually that's a good point. thanks. changed all the dates relative to start_date and end_date variables.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nice, looks like java integration tests failed though

)

entity_df = pd.DataFrame.from_dict(
{
# entity's join key -> entity values
Expand All @@ -97,17 +102,32 @@ def substrait_view(inputs: Table) -> Table:
}
)

requested_features = [
"driver_hourly_stats:conv_rate",
"driver_hourly_stats:acc_rate",
"driver_hourly_stats:avg_daily_trips",
"substrait_view:conv_rate_plus_acc_substrait",
"pandas_view:conv_rate_plus_acc",
]

training_df = store.get_historical_features(
entity_df=entity_df,
features=[
"driver_hourly_stats:conv_rate",
"driver_hourly_stats:acc_rate",
"driver_hourly_stats:avg_daily_trips",
"substrait_view:conv_rate_plus_acc_substrait",
"pandas_view:conv_rate_plus_acc",
],
).to_df()
entity_df=entity_df, features=requested_features
)

assert training_df.to_df()["conv_rate_plus_acc"].equals(
training_df.to_df()["conv_rate_plus_acc_substrait"]
)

assert training_df.to_arrow()["conv_rate_plus_acc"].equals(
training_df.to_arrow()["conv_rate_plus_acc_substrait"]
)

online_response = store.get_online_features(
features=requested_features,
entity_rows=[{"driver_id": 1001}, {"driver_id": 1002}, {"driver_id": 1003}],
)

assert training_df["conv_rate_plus_acc"].equals(
training_df["conv_rate_plus_acc_substrait"]
assert (
online_response.to_dict()["conv_rate_plus_acc"]
== online_response.to_dict()["conv_rate_plus_acc_substrait"]
)
Loading