feat: Add decimal datatype to spark mapping to feast data type

Signed-off-by: tanlocnguyen <[email protected]>
feast-dev · Apr 4, 2024 · 75ecfa6 · 75ecfa6
1 parent 6cba191
commit 75ecfa6
Show file tree

Hide file tree

Showing 84 changed files with 1,913 additions and 2,602 deletions.
diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml
@@ -167,4 +167,4 @@ jobs:
           SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }}
           SNOWFLAKE_CI_ROLE: ${{ secrets.SNOWFLAKE_CI_ROLE }}
           SNOWFLAKE_CI_WAREHOUSE: ${{ secrets.SNOWFLAKE_CI_WAREHOUSE }}
-        run: pytest -n 8 --cov=./ --cov-report=xml --color=yes sdk/python/tests --integration --durations=5 --timeout=1200 --timeout_method=thread
+        run: make test-python-integration
diff --git a/.github/workflows/pr_local_integration_tests.yml b/.github/workflows/pr_local_integration_tests.yml
@@ -61,9 +61,4 @@ jobs:
         run: make install-python-ci-dependencies
       - name: Test local integration tests
         if: ${{ always() }}  # this will guarantee that step won't be canceled and resources won't leak
-        env:
-          FEAST_USAGE: "False"
-          IS_TEST: "True"
-          FEAST_LOCAL_ONLINE_CONTAINER: "True"
-          FEAST_IS_LOCAL_TEST: "True"
-        run: pytest -n 8 --cov=./ --cov-report=xml --color=yes --integration -k "not gcs_registry and not s3_registry and not test_lambda_materialization and not test_snowflake_materialization" sdk/python/tests
+        run: make test-python-integration-local
diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml
@@ -23,17 +23,6 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
           architecture: x64
-      - name: Install mysql on macOS
-        if:  startsWith(matrix.os, 'macOS')
-        run: |
-          brew install mysql
-          PATH=$PATH:/usr/local/mysql/bin
-      - name: Work around Homebrew MySQL being broken
-        # See https://github.com/Homebrew/homebrew-core/issues/130258 for more details.
-        if:  startsWith(matrix.os, 'macOS')
-        run: |
-          brew install zlib
-          ln -sv $(brew --prefix zlib)/lib/libz.dylib $(brew --prefix)/lib/libzlib.dylib
       - name: Get pip cache dir
         id: pip-cache
         run: |
@@ -56,7 +45,7 @@ jobs:
       - name: Install dependencies
         run: make install-python-ci-dependencies
       - name: Test Python
-        run: pytest -n 8 --cov=./ --cov-report=xml --color=yes sdk/python/tests
+        run: make test-python-unit
 
 
   unit-test-ui:

diff --git a/Makefile b/Makefile
@@ -28,7 +28,7 @@ format: format-python format-java
 
 lint: lint-python lint-java
 
-test: test-python test-java
+test: test-python-unit test-java
 
 protos: compile-protos-python compile-protos-docs
 
@@ -63,32 +63,26 @@ benchmark-python:
 benchmark-python-local:
 	FEAST_USAGE=False IS_TEST=True FEAST_IS_LOCAL_TEST=True python -m pytest --integration --benchmark  --benchmark-autosave --benchmark-save-data sdk/python/tests
 
-test-python:
-	FEAST_USAGE=False \
-	IS_TEST=True \
-	python -m pytest -n 8 sdk/python/tests \
+test-python-unit:
+	python -m pytest -n 8 --color=yes sdk/python/tests
 
 test-python-integration:
-	FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration sdk/python/tests
+	python -m pytest -n 8 --integration -k "not minio_registry" --color=yes --durations=5 --timeout=1200 --timeout_method=thread sdk/python/tests
 
 test-python-integration-local:
 	@(docker info > /dev/null 2>&1 && \
-		FEAST_USAGE=False \
-		IS_TEST=True \
 		FEAST_IS_LOCAL_TEST=True \
 		FEAST_LOCAL_ONLINE_CONTAINER=True \
-		python -m pytest -n 8 --integration \
+		python -m pytest -n 8 --color=yes --integration \
 			-k "not gcs_registry and \
  				not s3_registry and \
  				not test_lambda_materialization and \
- 				not test_snowflake" \
+ 				not test_snowflake_materialization" \
 		sdk/python/tests \
 	) || echo "This script uses Docker, and it isn't running - please start the Docker Daemon and try again!";
 
 test-python-integration-container:
 	@(docker info > /dev/null 2>&1 && \
-		FEAST_USAGE=False \
-		IS_TEST=True \
 		FEAST_LOCAL_ONLINE_CONTAINER=True \
 		python -m pytest -n 8 --integration sdk/python/tests \
 	) || echo "This script uses Docker, and it isn't running - please start the Docker Daemon and try again!";
@@ -97,7 +91,6 @@ test-python-universal-spark:
 	PYTHONPATH='.' \
 	FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.spark_repo_configuration \
 	PYTEST_PLUGINS=feast.infra.offline_stores.contrib.spark_offline_store.tests \
- 	FEAST_USAGE=False IS_TEST=True \
  	python -m pytest -n 8 --integration \
  	 	-k "not test_historical_retrieval_fails_on_validation and \
 			not test_historical_retrieval_with_validation and \
@@ -121,7 +114,6 @@ test-python-universal-trino:
 	PYTHONPATH='.' \
 	FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.trino_repo_configuration \
 	PYTEST_PLUGINS=feast.infra.offline_stores.contrib.trino_offline_store.tests \
- 	FEAST_USAGE=False IS_TEST=True \
  	python -m pytest -n 8 --integration \
  	 	-k "not test_historical_retrieval_fails_on_validation and \
 			not test_historical_retrieval_with_validation and \
@@ -148,7 +140,6 @@ test-python-universal-mssql:
 	PYTHONPATH='.' \
 	FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.mssql_repo_configuration \
 	PYTEST_PLUGINS=feast.infra.offline_stores.contrib.mssql_offline_store.tests \
- 	FEAST_USAGE=False IS_TEST=True \
 	FEAST_LOCAL_ONLINE_CONTAINER=True \
  	python -m pytest -n 8 --integration \
  	 	-k "not gcs_registry and \
@@ -166,7 +157,6 @@ test-python-universal-athena:
 	PYTHONPATH='.' \
 	FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.athena_repo_configuration \
 	PYTEST_PLUGINS=feast.infra.offline_stores.contrib.athena_offline_store.tests \
- 	FEAST_USAGE=False IS_TEST=True \
 	ATHENA_REGION=ap-northeast-2 \
 	ATHENA_DATA_SOURCE=AwsDataCatalog \
 	ATHENA_DATABASE=default \
@@ -190,7 +180,6 @@ test-python-universal-athena:
 test-python-universal-duckdb:
 	PYTHONPATH='.' \
 	FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.duckdb_repo_configuration \
-	FEAST_USAGE=False IS_TEST=True \
 	python -m pytest -n 8 --integration \
 		-k "not test_nullable_online_store and \
 			not gcs_registry and \
@@ -204,8 +193,6 @@ test-python-universal-postgres-offline:
 	PYTHONPATH='.' \
 		FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.postgres_repo_configuration \
 		PYTEST_PLUGINS=sdk.python.feast.infra.offline_stores.contrib.postgres_offline_store.tests \
-		FEAST_USAGE=False \
-		IS_TEST=True \
 		python -m pytest -n 8 --integration \
  			-k "not test_historical_retrieval_with_validation and \
 				not test_historical_features_persisting and \
@@ -226,8 +213,6 @@ test-python-universal-postgres-online:
 	PYTHONPATH='.' \
 		FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.postgres_repo_configuration \
 		PYTEST_PLUGINS=sdk.python.feast.infra.offline_stores.contrib.postgres_offline_store.tests \
-		FEAST_USAGE=False \
-		IS_TEST=True \
 		python -m pytest -n 8 --integration \
  			-k "not test_universal_cli and \
  				not test_go_feature_server and \
@@ -247,8 +232,6 @@ test-python-universal-postgres-online:
 	PYTHONPATH='.' \
 		FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.mysql_repo_configuration \
 		PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.mysql \
-		FEAST_USAGE=False \
-		IS_TEST=True \
 		python -m pytest -n 8 --integration \
  			-k "not test_universal_cli and \
  				not test_go_feature_server and \
@@ -268,17 +251,13 @@ test-python-universal-cassandra:
 	PYTHONPATH='.' \
 	FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.cassandra_repo_configuration \
 	PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.cassandra \
-	FEAST_USAGE=False \
-	IS_TEST=True \
 	python -m pytest -x --integration \
 	sdk/python/tests
 
 test-python-universal-hazelcast:
 	PYTHONPATH='.' \
 		FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.hazelcast_repo_configuration \
 		PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.hazelcast \
-		FEAST_USAGE=False \
-		IS_TEST=True \
 		python -m pytest -n 8 --integration \
  			-k "not test_universal_cli and \
  				not test_go_feature_server and \
@@ -298,8 +277,6 @@ test-python-universal-cassandra-no-cloud-providers:
 	PYTHONPATH='.' \
 	FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.cassandra_repo_configuration \
 	PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.cassandra \
-	FEAST_USAGE=False \
-	IS_TEST=True \
 	python -m pytest -x --integration \
 	-k "not test_lambda_materialization_consistency   and \
 	  not test_apply_entity_integration               and \
@@ -314,7 +291,7 @@ test-python-universal-cassandra-no-cloud-providers:
 	sdk/python/tests
 
 test-python-universal:
-	FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration sdk/python/tests
+	python -m pytest -n 8 --integration sdk/python/tests
 
 format-python:
 	# Sort

diff --git a/java/CONTRIBUTING.md b/java/CONTRIBUTING.md
@@ -50,7 +50,7 @@ Automatically format the code to conform the style guide by:
 
 ```sh
 # formats all code in the feast-java repository
-mvn spotless:apply
+make format-java
 ```
 
 > If you're using IntelliJ, you can import these [code style settings](https://github.com/google/styleguide/blob/gh-pages/intellij-java-google-style.xml)
@@ -66,7 +66,7 @@ Run all Unit tests:
 make test-java
 ```
 
-Run all Integration tests (note: this also runs GCS + S3 based tests which should fail):
+Run all Integration tests:
 ```
 make test-java-integration
 ```

diff --git a/java/pom.xml b/java/pom.xml
@@ -68,6 +68,8 @@
         <google.auth.library.oauth2.http.version>0.21.0</google.auth.library.oauth2.http.version>
         <auto.value.version>1.6.6</auto.value.version>
         <guava.version>30.1-jre</guava.version>
+        <reactor.version>3.4.34</reactor.version>
+        <netty.version>4.1.101.Final</netty.version>
 
         <license.content><![CDATA[
 /*
@@ -186,6 +188,49 @@
                 <version>${javax.validation.version}</version>
             </dependency>
 
+            <dependency>
+                <groupId>com.fasterxml.jackson.core</groupId>
+                <artifactId>jackson-core</artifactId>
+                <version>${jackson.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>com.fasterxml.jackson.core</groupId>
+                <artifactId>jackson-databind</artifactId>
+                <version>${jackson.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>com.fasterxml.jackson.core</groupId>
+                <artifactId>jackson-annotations</artifactId>
+                <version>${jackson.version}</version>
+            </dependency>
+
+            <dependency>
+                <groupId>io.netty</groupId>
+                <artifactId>netty-common</artifactId>
+                <version>${netty.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>io.netty</groupId>
+                <artifactId>netty-buffer</artifactId>
+                <version>${netty.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>io.netty</groupId>
+                <artifactId>netty-handler</artifactId>
+                <version>${netty.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>io.netty</groupId>
+                <artifactId>netty-transport</artifactId>
+                <version>${netty.version}</version>
+            </dependency>
+
+            <dependency>
+                <groupId>io.projectreactor</groupId>
+                <artifactId>reactor-core</artifactId>
+                <version>${reactor.version}</version>
+            </dependency>
+
             <dependency>
                 <groupId>org.junit.platform</groupId>
                 <artifactId>junit-platform-engine</artifactId>
@@ -246,7 +291,7 @@
                 <configuration>
                     <java>
                         <licenseHeader>
-                          <content>${license.content}</content>
+                            <content>${license.content}</content>
                         </licenseHeader>
                         <googleJavaFormat>
                             <version>1.7</version>
@@ -264,15 +309,15 @@
                     </scala>
                 </configuration>
                 <executions>
-                  <!-- Move check to fail faster, but after compilation. Default is verify phase -->
-                  <execution>
-                      <id>spotless-check</id>
-                      <phase>process-test-classes</phase>
-                      <goals>
-                          <goal>check</goal>
-                      </goals>
-                  </execution>
-              </executions>
+                    <!-- Move check to fail faster, but after compilation. Default is verify phase -->
+                    <execution>
+                        <id>spotless-check</id>
+                        <phase>process-test-classes</phase>
+                        <goals>
+                            <goal>check</goal>
+                        </goals>
+                    </execution>
+                </executions>
             </plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>

diff --git a/java/serving/.gitignore b/java/serving/.gitignore
@@ -34,4 +34,7 @@ feast-serving.jar
 /.nb-gradle/
 
 ## Feast Temporary Files ##
-/temp/
+/temp/
+
+## Generated test data ##
+**/*.parquet
diff --git a/java/serving/pom.xml b/java/serving/pom.xml
@@ -16,8 +16,8 @@
   ~
   -->
 <project xmlns="http://maven.apache.org/POM/4.0.0"
-    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
   <modelVersion>4.0.0</modelVersion>
 
   <parent>
@@ -121,6 +121,19 @@
       <version>5.0.1</version>
     </dependency>
 
+    <!-- azure blob storage -->
+    <!-- https://mvnrepository.com/artifact/com.azure/azure-storage-blob -->
+    <dependency>
+      <groupId>com.azure</groupId>
+      <artifactId>azure-storage-blob</artifactId>
+      <version>12.25.2</version>
+    </dependency>
+    <dependency>
+      <groupId>com.azure</groupId>
+      <artifactId>azure-identity</artifactId>
+      <version>1.11.3</version>
+    </dependency>
+
     <!-- TODO: SLF4J is being used via Lombok, but also jog4j - pick one -->
     <dependency>
       <groupId>org.slf4j</groupId>
@@ -356,11 +369,11 @@
       <version>2.7.4</version>
       <scope>test</scope>
     </dependency>
-      <dependency>
-          <groupId>io.lettuce</groupId>
-          <artifactId>lettuce-core</artifactId>
-          <version>6.0.2.RELEASE</version>
-      </dependency>
+    <dependency>
+      <groupId>io.lettuce</groupId>
+      <artifactId>lettuce-core</artifactId>
+      <version>6.0.2.RELEASE</version>
+    </dependency>
     <dependency>
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-lang3</artifactId>