From 82e5a049ca27633bd3d74fae047ef42782c526f9 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Wed, 17 Jul 2024 04:44:07 -0700 Subject: [PATCH] fix(ingest/redshift): reduce severity of SQL parsing issues (#10924) --- .../src/datahub/ingestion/source/redshift/lineage_v2.py | 2 +- .../src/datahub/sql_parsing/sql_parsing_aggregator.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage_v2.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage_v2.py index 3d6c746183fd9..bd0bbe742a219 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage_v2.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage_v2.py @@ -419,7 +419,7 @@ def generate(self) -> Iterable[MetadataWorkUnit]: for mcp in self.aggregator.gen_metadata(): yield mcp.as_workunit() if len(self.aggregator.report.observed_query_parse_failures) > 0: - self.report.report_failure( + self.report.report_warning( title="Failed to extract some SQL lineage", message="Unexpected error(s) while attempting to extract lineage from SQL queries. See the full logs for more details.", context=f"Query Parsing Failures: {self.aggregator.report.observed_query_parse_failures}", diff --git a/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py b/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py index 677b96269fe58..894f01820718b 100644 --- a/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py +++ b/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py @@ -660,10 +660,10 @@ def add_observed_query( if parsed.debug_info.table_error: self.report.num_observed_queries_failed += 1 return # we can't do anything with this query - elif isinstance(parsed.debug_info.column_error, CooperativeTimeoutError): - self.report.num_observed_queries_column_timeout += 1 elif parsed.debug_info.column_error: self.report.num_observed_queries_column_failed += 1 + if isinstance(parsed.debug_info.column_error, CooperativeTimeoutError): + self.report.num_observed_queries_column_timeout += 1 query_fingerprint = parsed.query_fingerprint