Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Rename Metadata Store to Registry #1433

Merged
merged 2 commits into from
Apr 3, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions sdk/python/feast/feature_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,18 +65,18 @@ def __init__(
self.config = load_repo_config(Path(repo_path))
else:
self.config = RepoConfig(
metadata_store="./metadata.db",
registry="./registry.db",
project="default",
provider="local",
online_store=OnlineStoreConfig(
local=LocalOnlineStoreConfig(path="online_store.db")
),
)

metadata_store_config = self.config.get_metadata_store_config()
registry_config = self.config.get_registry_config()
self._registry = Registry(
registry_path=metadata_store_config.path,
cache_ttl=timedelta(seconds=metadata_store_config.cache_ttl_seconds),
registry_path=registry_config.path,
cache_ttl=timedelta(seconds=registry_config.cache_ttl_seconds),
)

@property
Expand All @@ -101,10 +101,10 @@ def refresh_registry(self):
downloaded synchronously, which may increase latencies if the triggering method is get_online_features()
"""

metadata_store_config = self.config.get_metadata_store_config()
registry_config = self.config.get_registry_config()
self._registry = Registry(
registry_path=metadata_store_config.path,
cache_ttl=timedelta(seconds=metadata_store_config.cache_ttl_seconds),
registry_path=registry_config.path,
cache_ttl=timedelta(seconds=registry_config.cache_ttl_seconds),
)
self._registry.refresh()

Expand Down
14 changes: 7 additions & 7 deletions sdk/python/feast/repo_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ class OnlineStoreConfig(FeastBaseModel):
""" LocalOnlineStoreConfig: Optional local online store config """


class MetadataStoreConfig(FeastBaseModel):
class RegistryConfig(FeastBaseModel):
""" Metadata Store Configuration. Configuration that relates to reading from and writing to the Feast registry."""

path: StrictStr
Expand All @@ -51,7 +51,7 @@ class MetadataStoreConfig(FeastBaseModel):
class RepoConfig(FeastBaseModel):
""" Repo config. Typically loaded from `feature_store.yaml` """

metadata_store: Union[StrictStr, MetadataStoreConfig]
registry: Union[StrictStr, RegistryConfig]
""" str: Path to metadata store. Can be a local path, or remote object storage path, e.g. gcs://foo/bar """

project: StrictStr
Expand All @@ -66,11 +66,11 @@ class RepoConfig(FeastBaseModel):
online_store: Optional[OnlineStoreConfig] = None
""" OnlineStoreConfig: Online store configuration (optional depending on provider) """

def get_metadata_store_config(self):
if isinstance(self.metadata_store, str):
return MetadataStoreConfig(path=self.metadata_store)
def get_registry_config(self):
if isinstance(self.registry, str):
return RegistryConfig(path=self.registry)
else:
return self.metadata_store
return self.registry


# This is the JSON Schema for config validation. We use this to have nice detailed error messages
Expand All @@ -84,7 +84,7 @@ def get_metadata_store_config(self):
"type": "object",
"properties": {
"project": {"type": "string"},
"metadata_store": {"type": "string"},
"registry": {"type": "string"},
"provider": {"type": "string"},
"online_store": {
"type": "object",
Expand Down
24 changes: 12 additions & 12 deletions sdk/python/feast/repo_operations.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,11 +58,11 @@ def parse_repo(repo_root: Path) -> ParsedRepo:
def apply_total(repo_config: RepoConfig, repo_path: Path):
os.chdir(repo_path)
sys.path.append("")
metadata_store_config = repo_config.get_metadata_store_config()
registry_config = repo_config.get_registry_config()
project = repo_config.project
registry = Registry(
registry_path=metadata_store_config.path,
cache_ttl=timedelta(seconds=metadata_store_config.cache_ttl_seconds),
registry_path=registry_config.path,
cache_ttl=timedelta(seconds=registry_config.cache_ttl_seconds),
)
repo = parse_repo(repo_path)

Expand Down Expand Up @@ -121,10 +121,10 @@ def apply_total(repo_config: RepoConfig, repo_path: Path):


def teardown(repo_config: RepoConfig, repo_path: Path):
metadata_store_config = repo_config.get_metadata_store_config()
registry_config = repo_config.get_registry_config()
registry = Registry(
registry_path=metadata_store_config.path,
cache_ttl=timedelta(seconds=metadata_store_config.cache_ttl_seconds),
registry_path=registry_config.path,
cache_ttl=timedelta(seconds=registry_config.cache_ttl_seconds),
)
project = repo_config.project
registry_tables: List[Union[FeatureTable, FeatureView]] = []
Expand All @@ -136,11 +136,11 @@ def teardown(repo_config: RepoConfig, repo_path: Path):

def registry_dump(repo_config: RepoConfig):
""" For debugging only: output contents of the metadata registry """
metadata_store_config = repo_config.get_metadata_store_config()
registry_config = repo_config.get_registry_config()
project = repo_config.project
registry = Registry(
registry_path=metadata_store_config.path,
cache_ttl=timedelta(seconds=metadata_store_config.cache_ttl_seconds),
registry_path=registry_config.path,
cache_ttl=timedelta(seconds=registry_config.cache_ttl_seconds),
)

for entity in registry.list_entities(project=project):
Expand Down Expand Up @@ -173,7 +173,7 @@ def init_repo(repo_path: Path, minimal: bool):
dedent(
f"""
project: {project_id}
metadata_store: /path/to/metadata.db
registry: /path/to/registry.db
provider: local
online_store:
local:
Expand All @@ -182,7 +182,7 @@ def init_repo(repo_path: Path, minimal: bool):
)
)
print(
"Generated example feature_store.yaml. Please edit metadata_store and online_store"
"Generated example feature_store.yaml. Please edit registry and online_store"
"location before running apply"
)

Expand Down Expand Up @@ -211,7 +211,7 @@ def init_repo(repo_path: Path, minimal: bool):
dedent(
f"""
project: {project_id}
metadata_store: {"data/metadata.db"}
registry: {"data/registry.db"}
provider: local
online_store:
local:
Expand Down
2 changes: 1 addition & 1 deletion sdk/python/tests/cli_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ def local_repo(self, example_repo_py: str):
dedent(
f"""
project: {project_id}
metadata_store: {data_path / "metadata.db"}
registry: {data_path / "registry.db"}
provider: local
online_store:
local:
Expand Down
2 changes: 1 addition & 1 deletion sdk/python/tests/online_write_benchmark.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ def benchmark_writes():
with tempfile.TemporaryDirectory() as temp_dir:
store = FeatureStore(
config=RepoConfig(
metadata_store=os.path.join(temp_dir, "metadata.db"),
registry=os.path.join(temp_dir, "registry.db"),
project=project_id,
provider="gcp",
)
Expand Down
2 changes: 1 addition & 1 deletion sdk/python/tests/test_cli_gcp.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ def test_basic() -> None:
dedent(
f"""
project: {project_id}
metadata_store: {data_path / "metadata.db"}
registry: {data_path / "registry.db"}
provider: gcp
"""
)
Expand Down
2 changes: 1 addition & 1 deletion sdk/python/tests/test_cli_local.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ def test_workflow() -> None:
dedent(
f"""
project: foo
metadata_store: {data_path / "metadata.db"}
registry: {data_path / "registry.db"}
provider: local
online_store:
local:
Expand Down
6 changes: 3 additions & 3 deletions sdk/python/tests/test_feature_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ def feature_store_with_local_registry(self):
fd, online_store_path = mkstemp()
return FeatureStore(
config=RepoConfig(
metadata_store=registry_path,
registry=registry_path,
project="default",
provider="local",
online_store=OnlineStoreConfig(
Expand All @@ -57,11 +57,11 @@ def feature_store_with_gcs_registry(self):
age=14
) # delete buckets automatically after 14 days
bucket.patch()
bucket.blob("metadata.db")
bucket.blob("registry.db")

return FeatureStore(
config=RepoConfig(
metadata_store=f"gs://{bucket_name}/metadata.db",
registry=f"gs://{bucket_name}/registry.db",
project="default",
provider="gcp",
)
Expand Down
4 changes: 2 additions & 2 deletions sdk/python/tests/test_historical_retrieval.py
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,7 @@ def test_historical_features_from_parquet_sources():

store = FeatureStore(
config=RepoConfig(
metadata_store=os.path.join(temp_dir, "metadata.db"),
registry=os.path.join(temp_dir, "registry.db"),
project="default",
provider="local",
online_store=OnlineStoreConfig(
Expand Down Expand Up @@ -331,7 +331,7 @@ def test_historical_features_from_bigquery_sources():

store = FeatureStore(
config=RepoConfig(
metadata_store=os.path.join(temp_dir, "metadata.db"),
registry=os.path.join(temp_dir, "registry.db"),
project="default",
provider="local",
online_store=OnlineStoreConfig(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ def test_bigquery_table_to_datastore_correctness(self):
),
)
config = RepoConfig(
metadata_store="./metadata.db",
registry="./registry.db",
project=f"test_bq_table_correctness_{int(time.time())}",
provider="gcp",
)
Expand Down Expand Up @@ -133,7 +133,7 @@ def test_bigquery_query_to_datastore_correctness(self):
),
)
config = RepoConfig(
metadata_store="./metadata.db",
registry="./registry.db",
project=f"test_bq_query_correctness_{int(time.time())}",
provider="gcp",
)
Expand Down
26 changes: 13 additions & 13 deletions sdk/python/tests/test_online_retrieval.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
from feast import FeatureStore, RepoConfig
from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto
from feast.protos.feast.types.Value_pb2 import Value as ValueProto
from feast.repo_config import MetadataStoreConfig
from feast.repo_config import RegistryConfig
from tests.cli_utils import CliRunner, get_example_repo


Expand Down Expand Up @@ -85,8 +85,8 @@ def test_online() -> None:
cache_ttl = 1
fs_fast_ttl = FeatureStore(
config=RepoConfig(
metadata_store=MetadataStoreConfig(
path=store.config.metadata_store, cache_ttl_seconds=cache_ttl
registry=RegistryConfig(
path=store.config.registry, cache_ttl_seconds=cache_ttl
),
online_store=store.config.online_store,
project=store.config.project,
Expand All @@ -101,8 +101,8 @@ def test_online() -> None:
)
assert result.to_dict()["driver_locations:lon"] == ["1.0", None]

# Rename the metadata.db so that it cant be used for refreshes
os.rename(store.config.metadata_store, store.config.metadata_store + "_fake")
# Rename the registry.db so that it cant be used for refreshes
os.rename(store.config.registry, store.config.registry + "_fake")

# Wait for registry to expire
time.sleep(cache_ttl)
Expand All @@ -114,8 +114,8 @@ def test_online() -> None:
entity_rows=[{"driver": 1}, {"driver": 123}],
)

# Restore metadata.db so that we can see if it actually reloads registry
os.rename(store.config.metadata_store + "_fake", store.config.metadata_store)
# Restore registry.db so that we can see if it actually reloads registry
os.rename(store.config.registry + "_fake", store.config.registry)

# Test if registry is actually reloaded and whether results return
result = fs_fast_ttl.get_online_features(
Expand All @@ -127,8 +127,8 @@ def test_online() -> None:
# Create a registry with infinite cache (for users that want to manually refresh the registry)
fs_infinite_ttl = FeatureStore(
config=RepoConfig(
metadata_store=MetadataStoreConfig(
path=store.config.metadata_store, cache_ttl_seconds=0
registry=RegistryConfig(
path=store.config.registry, cache_ttl_seconds=0
),
online_store=store.config.online_store,
project=store.config.project,
Expand All @@ -146,8 +146,8 @@ def test_online() -> None:
# Wait a bit so that an arbitrary TTL would take effect
time.sleep(2)

# Rename the metadata.db so that it cant be used for refreshes
os.rename(store.config.metadata_store, store.config.metadata_store + "_fake")
# Rename the registry.db so that it cant be used for refreshes
os.rename(store.config.registry, store.config.registry + "_fake")

# TTL is infinite so this method should use registry cache
result = fs_infinite_ttl.get_online_features(
Expand All @@ -160,5 +160,5 @@ def test_online() -> None:
with pytest.raises(FileNotFoundError):
fs_infinite_ttl.refresh_registry()

# Restore metadata.db so that teardown works
os.rename(store.config.metadata_store + "_fake", store.config.metadata_store)
# Restore registry.db so that teardown works
os.rename(store.config.registry + "_fake", store.config.registry)
10 changes: 5 additions & 5 deletions sdk/python/tests/test_repo_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ def test_basic(self) -> None:
dedent(
"""
project: foo
metadata_store: "metadata.db"
registry: "registry.db"
provider: local
online_store:
local:
Expand All @@ -48,7 +48,7 @@ def test_basic(self) -> None:
dedent(
"""
project: foo
metadata_store: "metadata.db"
registry: "registry.db"
provider: gcp
"""
),
Expand All @@ -60,7 +60,7 @@ def test_errors(self) -> None:
dedent(
"""
project: foo
metadata_store: "metadata.db"
registry: "registry.db"
provider: local
online_store:
local:
Expand All @@ -76,7 +76,7 @@ def test_errors(self) -> None:
dedent(
"""
project: foo
metadata_store: "metadata.db"
registry: "registry.db"
provider: local
online_store:
local:
Expand All @@ -91,7 +91,7 @@ def test_errors(self) -> None:
self._test_config(
dedent(
"""
metadata_store: "metadata.db"
registry: "registry.db"
provider: local
online_store:
local:
Expand Down