Skip to content

Commit

Permalink
squashing the last 15 commits to one.
Browse files Browse the repository at this point in the history
  • Loading branch information
lokeshrangineni committed Jun 12, 2024
1 parent a75cd1e commit d895fdc
Show file tree
Hide file tree
Showing 9 changed files with 151 additions and 284 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/pr_integration_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ jobs:
strategy:
fail-fast: false
matrix:
python-version: [ "3.9", "3.10", "3.11" ]
python-version: [ "3.11" ]
os: [ ubuntu-latest ]
env:
OS: ${{ matrix.os }}
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/pr_local_integration_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ jobs:
strategy:
fail-fast: false
matrix:
python-version: [ "3.9", "3.10", "3.11" ]
python-version: [ "3.11" ]
os: [ ubuntu-latest ]
env:
OS: ${{ matrix.os }}
Expand Down
6 changes: 0 additions & 6 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -70,12 +70,6 @@ lock-python-dependencies-all:
pixi run --environment py311 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.11-requirements.txt"
pixi run --environment py311 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.11-ci-requirements.txt"

lock-python-dependencies-all:
pixi run --environment py39 --manifest-path infra/scripts/pixi/pixi.toml "python -m piptools compile -U --output-file sdk/python/requirements/py3.9-requirements.txt"
pixi run --environment py39 --manifest-path infra/scripts/pixi/pixi.toml "python -m piptools compile -U --extra ci --output-file sdk/python/requirements/py3.9-ci-requirements.txt"
pixi run --environment py310 --manifest-path infra/scripts/pixi/pixi.toml "python -m piptools compile -U --output-file sdk/python/requirements/py3.10-requirements.txt"
pixi run --environment py310 --manifest-path infra/scripts/pixi/pixi.toml "python -m piptools compile -U --extra ci --output-file sdk/python/requirements/py3.10-ci-requirements.txt"

benchmark-python:
IS_TEST=True python -m pytest --integration --benchmark --benchmark-autosave --benchmark-save-data sdk/python/tests

Expand Down
366 changes: 120 additions & 246 deletions infra/scripts/pixi/pixi.lock

Large diffs are not rendered by default.

8 changes: 6 additions & 2 deletions infra/scripts/pixi/pixi.toml
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,18 @@ platforms = ["linux-64"]
[tasks]

[dependencies]
pip-tools = ">=7.4.1,<7.5"
uv = ">=0.1.39,<0.2"

[feature.py39.dependencies]
python = "~=3.9.0"

[feature.py310.dependencies]
python = "~=3.10.0"

[feature.py311.dependencies]
python = "~=3.11.0"

[environments]
py39 = ["py39"]
py310 = ["py310"]
py310 = ["py310"]
py311 = ["py311"]
24 changes: 8 additions & 16 deletions sdk/python/feast/infra/online_stores/remote.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@


class RemoteOnlineStoreConfig(FeastConfigBaseModel):
"""Online store config for Redis store"""
"""Remote Online store config for remote online store"""

type: Literal["remote"] = "remote"
"""Online store type selector"""
Expand Down Expand Up @@ -78,27 +78,23 @@ def online_read(
response_json = json.loads(response.text)
event_ts = self._get_event_ts(response_json)
# Iterating over results and converting the API results in column format to row format.
feature_value_index = 0
result_tuples: List[
Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]
] = []
while feature_value_index < len(entity_keys):
feature_values_dict = dict()
for feature_value_index in range(len(entity_keys)):
feature_values_dict: Dict[str, ValueProto] = dict()
for index, feature_name in enumerate(
response_json["metadata"]["feature_names"]
):
if self._check_if_feature_requested(
feature_name, requested_features
):
is_present = (
True
if response_json["results"][index]["statuses"][
if (
response_json["results"][index]["statuses"][
feature_value_index
]
== "PRESENT"
else False
)
if is_present:
):
message = python_values_to_proto_values(
[
response_json["results"][index]["values"][
Expand All @@ -112,7 +108,6 @@ def online_read(
feature_values_dict[feature_name] = ValueProto()

result_tuples.append((event_ts, feature_values_dict))
feature_value_index += 1
return result_tuples
else:
error_msg = f"Unable to retrieve the online store data using feature server API. Error_code={response.status_code}, error_message={response.reason}"
Expand Down Expand Up @@ -154,10 +149,8 @@ def _check_if_feature_requested(self, feature_name, requested_features):

def _get_event_ts(self, response_json) -> datetime:
event_ts = ""
for index, result in enumerate(response_json["results"]):
if index == 1:
event_ts = result["event_timestamps"][0]
break
if len(response_json["results"]) > 1:
event_ts = response_json["results"][1]["event_timestamps"][0]
return datetime.fromisoformat(event_ts.replace("Z", "+00:00"))

def update(
Expand All @@ -170,7 +163,6 @@ def update(
partial: bool,
):
pass
# raise NotImplementedError("for remote online store update method is not available.")

def teardown(
self,
Expand Down
17 changes: 7 additions & 10 deletions sdk/python/tests/integration/feature_repos/repo_configuration.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@
from tests.integration.feature_repos.universal.data_sources.file import (
DuckDBDataSourceCreator,
DuckDBDeltaDataSourceCreator,
DuckDBDeltaS3DataSourceCreator,
FileDataSourceCreator,
)
from tests.integration.feature_repos.universal.data_sources.redshift import (
Expand Down Expand Up @@ -80,7 +79,6 @@
"connection_string": "127.0.0.1:6001,127.0.0.1:6002,127.0.0.1:6003",
}


SNOWFLAKE_CONFIG = {
"type": "snowflake.online",
"account": os.getenv("SNOWFLAKE_CI_DEPLOYMENT", ""),
Expand Down Expand Up @@ -153,7 +151,6 @@
AVAILABLE_ONLINE_STORES["datastore"] = ("datastore", None)
AVAILABLE_ONLINE_STORES["snowflake"] = (SNOWFLAKE_CONFIG, None)
AVAILABLE_ONLINE_STORES["bigtable"] = (BIGTABLE_CONFIG, None)

# Uncomment to test using private Rockset account. Currently not enabled as
# there is no dedicated Rockset instance for CI testing and there is no
# containerized version of Rockset.
Expand Down Expand Up @@ -468,6 +465,13 @@ def construct_test_environment(
project, fixture_request=fixture_request
)

if test_repo_config.online_store_creator:
online_creator = test_repo_config.online_store_creator(
project, fixture_request=fixture_request
)
else:
online_creator = None

if test_repo_config.python_feature_server and test_repo_config.provider == "aws":
from feast.infra.feature_servers.aws_lambda.config import (
AwsLambdaFeatureServerConfig,
Expand Down Expand Up @@ -501,13 +505,6 @@ def construct_test_environment(
cache_ttl_seconds=1,
)

if test_repo_config.online_store_creator:
online_creator = test_repo_config.online_store_creator(
project, fixture_request=fixture_request, registry_path=registry.path
)
else:
online_creator = None

environment = Environment(
name=project,
provider=test_repo_config.provider,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ def test_remote_online_store_read():
server_store, server_url, registry_path = (
_create_server_store_spin_feature_server(temp_dir=remote_server_tmp_dir)
)

assert None not in (server_store, server_url, registry_path)
client_store = _create_remote_client_feature_store(
temp_dir=remote_client_tmp_dir,
Expand Down Expand Up @@ -100,6 +99,14 @@ def _assert_existing_feature_views_entity(
entity_rows=entity_rows,
)

features = ["driver_hourly_stats:conv_rate"]
_assert_client_server_online_stores_are_matching(
client_store=client_store,
server_store=server_store,
features=features,
entity_rows=entity_rows,
)


def _assert_client_server_online_stores_are_matching(
client_store: FeatureStore, server_store: FeatureStore, features, entity_rows
Expand Down
1 change: 0 additions & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,6 @@
"uvicorn[standard]>=0.14.0,<1",
"gunicorn; platform_system != 'Windows'",
"dask[dataframe]>=2024.4.2",
"bowler", # Needed for automatic repo upgrades
]

GCP_REQUIRED = [
Expand Down

0 comments on commit d895fdc

Please sign in to comment.