fix(ingest/redshift): reduce severity of SQL parsing issues (#10924)

datahub-project · Jul 17, 2024 · 82e5a04 · 82e5a04
1 parent 13b6feb
commit 82e5a04
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 3 deletions.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage_v2.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage_v2.py
@@ -419,7 +419,7 @@ def generate(self) -> Iterable[MetadataWorkUnit]:
         for mcp in self.aggregator.gen_metadata():
             yield mcp.as_workunit()
         if len(self.aggregator.report.observed_query_parse_failures) > 0:
-            self.report.report_failure(
+            self.report.report_warning(
                 title="Failed to extract some SQL lineage",
                 message="Unexpected error(s) while attempting to extract lineage from SQL queries. See the full logs for more details.",
                 context=f"Query Parsing Failures: {self.aggregator.report.observed_query_parse_failures}",

diff --git a/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py b/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py
@@ -660,10 +660,10 @@ def add_observed_query(
         if parsed.debug_info.table_error:
             self.report.num_observed_queries_failed += 1
             return  # we can't do anything with this query
-        elif isinstance(parsed.debug_info.column_error, CooperativeTimeoutError):
-            self.report.num_observed_queries_column_timeout += 1
         elif parsed.debug_info.column_error:
             self.report.num_observed_queries_column_failed += 1
+            if isinstance(parsed.debug_info.column_error, CooperativeTimeoutError):
+                self.report.num_observed_queries_column_timeout += 1
 
         query_fingerprint = parsed.query_fingerprint