fix: Handle complex Spark data types in SparkSource (#3154)

* Make sure data types are strings Signed-off-by: Petra Rebernjak <[email protected]> * Lint Signed-off-by: Petra Rebernjak <[email protected]> Signed-off-by: Petra Rebernjak <[email protected]>
feast-dev · Aug 31, 2022 · 0e13573 · 0e13573
1 parent 1ee39fe
commit 0e13573
Showing 1 changed file with 1 addition and 4 deletions.
diff --git a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark_source.py b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark_source.py
@@ -159,10 +159,7 @@ def get_table_column_names_and_types(
             store_config=config.offline_store
         )
         df = spark_session.sql(f"SELECT * FROM {self.get_table_query_string()}")
-        return (
-            (fields["name"], fields["type"])
-            for fields in df.schema.jsonValue()["fields"]
-        )
+        return ((field.name, field.dataType.simpleString()) for field in df.schema)
 
     def get_table_query_string(self) -> str:
         """Returns a string that can directly be used to reference this table in SQL"""