feat: Snowflake skip materialization if no table change (feast-dev#3404)

* feat: Snowflake skip materialization if no table change Signed-off-by: Katy Haynie <[email protected]> * removing print statement from offline snowflake.py Signed-off-by: Katy Haynie <[email protected]> Signed-off-by: Katy Haynie <[email protected]>
franciscojavierarceo · Dec 20, 2022 · 0ab3942 · 0ab3942
1 parent 9d33daf
commit 0ab3942
Showing 1 changed file with 13 additions and 0 deletions.
diff --git a/sdk/python/feast/infra/materialization/snowflake_engine.py b/sdk/python/feast/infra/materialization/snowflake_engine.py
@@ -8,6 +8,7 @@
 import pandas as pd
 from colorama import Fore, Style
 from pydantic import Field, StrictStr
+from pytz import utc
 from tqdm import tqdm
 
 import feast
@@ -256,6 +257,18 @@ def _materialize_one(
                 end_date=end_date,
             )
 
+            # Lets check and see if we can skip this query, because the table hasnt changed
+            # since before the start date of this query
+            with get_snowflake_conn(self.repo_config.offline_store) as conn:
+                query = f"""SELECT SYSTEM$LAST_CHANGE_COMMIT_TIME('{feature_view.batch_source.get_table_query_string()}') AS last_commit_change_time"""
+                last_commit_change_time = (
+                    conn.cursor().execute(query).fetchall()[0][0] / 1_000_000_000
+                )
+            if last_commit_change_time < start_date.astimezone(tz=utc).timestamp():
+                return SnowflakeMaterializationJob(
+                    job_id=job_id, status=MaterializationJobStatus.SUCCEEDED
+                )
+
             fv_latest_values_sql = offline_job.to_sql()
 
             if feature_view.batch_source.field_mapping is not None: