From 9b430ca779b93018f7ab3e9eb900fcbec3f592f0 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Wed, 10 Nov 2021 17:19:22 -0800 Subject: [PATCH] Remove duplicates in setup.py and run rudimentary verifications (#2016) * Remove duplicates in setup.py Signed-off-by: Achal Shah * move pip-tools Signed-off-by: Achal Shah * Add step Signed-off-by: Achal Shah * use extend Signed-off-by: Achal Shah * remove ~ Signed-off-by: Achal Shah * dev_required Signed-off-by: Achal Shah * break deps Signed-off-by: Achal Shah * unbreak Signed-off-by: Achal Shah * Add proto-plus Signed-off-by: Achal Shah --- .github/workflows/unit_tests.yml | 2 ++ CONTRIBUTING.md | 2 +- README.md | 25 +++++++++++++++---------- sdk/python/setup.py | 16 ++++++---------- 4 files changed, 24 insertions(+), 21 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index dd01a63a5e..9a842a1675 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -41,6 +41,8 @@ jobs: run: make install-python-ci-dependencies - name: Test Python run: FEAST_USAGE=False pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests + - name: Ensure conflict-free dependencies + run: FEAST_USAGE=False pip-compile --dry-run sdk/python/setup.py --extra ci - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 with: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7afa98033a..c7a8e2430f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -59,7 +59,7 @@ pip install --upgrade pip 4. Install development dependencies for Feast Python SDK / CLI ```sh -pip install -e "sdk/python[ci]" +pip install -e "sdk/python[dev]" ``` ### Code Style & Linting diff --git a/README.md b/README.md index 91f87b52ed..42e2553d5e 100644 --- a/README.md +++ b/README.md @@ -137,31 +137,36 @@ The list below contains the functionality that contributors are planning to deve * [x] [Redshift source](https://docs.feast.dev/reference/data-sources/redshift) * [x] [BigQuery source](https://docs.feast.dev/reference/data-sources/bigquery) * [x] [Parquet file source](https://docs.feast.dev/reference/data-sources/file) - * [ ] Kafka source (Planned for Q4 2021) - * [ ] Synapse source (Planned for Q4 2021) + * [x] [Synapse source (community plugin)](https://github.com/Azure/feast-azure) + * [x] [Hive (community plugin)](https://github.com/baineng/feast-hive) + * [x] [Postgres (community plugin)](https://github.com/nossrannug/feast-postgres) + * [x] Kafka source (with [push support into the online store](reference/alpha-stream-ingestion.md)) * [ ] Snowflake source (Planned for Q4 2021) * [ ] HTTP source * **Offline Stores** * [x] [Redshift](https://docs.feast.dev/reference/offline-stores/redshift) * [x] [BigQuery](https://docs.feast.dev/reference/offline-stores/bigquery) + * [x] [Synapse (community plugin)](https://github.com/Azure/feast-azure) + * [x] [Hive (community plugin)](https://github.com/baineng/feast-hive) + * [x] [Postgres (community plugin)](https://github.com/nossrannug/feast-postgres) * [x] [In-memory / Pandas](https://docs.feast.dev/reference/offline-stores/file) * [x] [Custom offline store support](https://docs.feast.dev/how-to-guides/adding-a-new-offline-store) - * [x] [Hive (community maintained)](https://github.com/baineng/feast-hive) - * [x] [Postgres (community maintained)](https://github.com/nossrannug/feast-postgres) * [ ] Snowflake (Planned for Q4 2021) - * [ ] Synapse (Planned for Q4 2021) + * [ ] Trino (Planned for Q4 2021) * **Online Stores** * [x] [DynamoDB](https://docs.feast.dev/reference/online-stores/dynamodb) * [x] [Redis](https://docs.feast.dev/reference/online-stores/redis) * [x] [Datastore](https://docs.feast.dev/reference/online-stores/datastore) * [x] [SQLite](https://docs.feast.dev/reference/online-stores/sqlite) + * [x] [Azure Cache for Redis (community plugin)](https://github.com/Azure/feast-azure) + * [x] [Postgres (community plugin)](https://github.com/nossrannug/feast-postgres) * [x] [Custom online store support](https://docs.feast.dev/how-to-guides/adding-support-for-a-new-online-store) - * [x] [Postgres (community maintained)](https://github.com/nossrannug/feast-postgres) * [ ] Bigtable - * [ ] Cassandra\\ + * [ ] Cassandra * **Streaming** * [x] [Custom streaming ingestion job support](https://docs.feast.dev/how-to-guides/creating-a-custom-provider) - * [ ] Streaming ingestion on AWS (Planned for Q4 2021) + * [x] [Push based streaming data ingestion](reference/alpha-stream-ingestion.md) + * [ ] Streaming ingestion on AWS * [ ] Streaming ingestion on GCP * **Feature Engineering** * [x] On-demand Transformations (Alpha release. See [RFC](https://docs.google.com/document/d/1lgfIw0Drc65LpaxbUu49RCeJgMew547meSJttnUqz7c/edit#)) @@ -171,14 +176,14 @@ The list below contains the functionality that contributors are planning to deve * [x] AWS Lambda (Alpha release. See [RFC](https://docs.google.com/document/d/1eZWKWzfBif66LDN32IajpaG-j82LSHCCOzY6R7Ax7MI/edit)) * [ ] Cloud Run * [ ] Kubernetes - * [ ] KNative\\ + * [ ] KNative * **Feature Serving** * [x] Python Client * [x] REST Feature Server (Python) (Alpha release. See [RFC](https://docs.google.com/document/d/1iXvFhAsJ5jgAhPOpTdB3j-Wj1S9x3Ev\_Wr6ZpnLzER4/edit)) * [x] gRPC Feature Server (Java) (See [#1497](https://github.com/feast-dev/feast/issues/1497)) + * [x] Push API * [ ] Java Client * [ ] Go Client - * [ ] Push API * [ ] Delete API * [ ] Feature Logging (for training) * **Data Quality Management** diff --git a/sdk/python/setup.py b/sdk/python/setup.py index b5eeeecedf..eb20c4f594 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -54,6 +54,7 @@ "pandas>=1.0.0", "pandavro==1.5.*", "protobuf>=3.10", + "proto-plus", "pyarrow>=4.0.0", "pydantic>=1.0.0", "PyYAML>=5.4.*", @@ -113,15 +114,10 @@ "firebase-admin==4.5.2", "pre-commit", "assertpy==1.1", - "proto-plus<1.19.7", - "google-cloud-bigquery>=2.28.1", - "google-cloud-bigquery-storage >= 2.0.0", - "google-cloud-datastore>=2.1.*", - "google-cloud-storage>=1.20.*,<1.41", - "google-cloud-core==1.4.*", - "redis-py-cluster==2.1.2", - "boto3==1.17.*", -] + "pip-tools" +] + GCP_REQUIRED + REDIS_REQUIRED + AWS_REQUIRED + +DEV_REQUIRED = ["mypy-protobuf==1.*", "grpcio-testing==1.*"] + CI_REQUIRED # Get git repo root directory repo_root = str(pathlib.Path(__file__).resolve().parent.parent.parent) @@ -215,7 +211,7 @@ def run(self): # https://stackoverflow.com/questions/28509965/setuptools-development-requirements # Install dev requirements with: pip install -e .[dev] extras_require={ - "dev": ["mypy-protobuf==1.*", "grpcio-testing==1.*"], + "dev": DEV_REQUIRED, "ci": CI_REQUIRED, "gcp": GCP_REQUIRED, "aws": AWS_REQUIRED,