-
Notifications
You must be signed in to change notification settings - Fork 3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(ingest): bigquery-beta - Parsing view ddl definition for lineage (…
…#6187) Co-authored-by: Shirshanka Das <[email protected]>
- Loading branch information
1 parent
c7cc0af
commit 6e34cd6
Showing
5 changed files
with
176 additions
and
8 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,87 @@ | ||
import datetime | ||
|
||
from datahub.ingestion.source.bigquery_v2.bigquery_config import BigQueryV2Config | ||
from datahub.ingestion.source.bigquery_v2.bigquery_report import BigQueryV2Report | ||
from datahub.ingestion.source.bigquery_v2.bigquery_schema import BigqueryView | ||
from datahub.ingestion.source.bigquery_v2.lineage import BigqueryLineageExtractor | ||
|
||
|
||
def test_parse_view_lineage(): | ||
config = BigQueryV2Config() | ||
report = BigQueryV2Report() | ||
extractor = BigqueryLineageExtractor(config, report) | ||
|
||
# ddl = "select * from some_dataset.sometable as a" | ||
ddl = """CREATE VIEW `my-project.my-dataset.test_table` | ||
AS SELECT | ||
* REPLACE( | ||
myrandom(something) AS something) | ||
FROM | ||
`my-project2.my-dataset2.test_physical_table`; | ||
""" | ||
view = BigqueryView( | ||
name="test", | ||
created=datetime.datetime.now(), | ||
last_altered=datetime.datetime.now(), | ||
comment="", | ||
ddl=ddl, | ||
) | ||
tables = extractor.parse_view_lineage("my_project", "my_dataset", view) | ||
assert 1 == len(tables) | ||
assert "my-project2.my-dataset2.test_physical_table" == tables[0].get_table_name() | ||
|
||
|
||
def test_parse_view_lineage_with_two_part_table_name(): | ||
config = BigQueryV2Config() | ||
report = BigQueryV2Report() | ||
extractor = BigqueryLineageExtractor(config, report) | ||
|
||
ddl = "CREATE VIEW my_view as select * from some_dataset.sometable as a" | ||
view = BigqueryView( | ||
name="test", | ||
created=datetime.datetime.now(), | ||
last_altered=datetime.datetime.now(), | ||
comment="", | ||
ddl=ddl, | ||
) | ||
tables = extractor.parse_view_lineage("my_project", "my_dataset", view) | ||
assert 1 == len(tables) | ||
assert "my_project.some_dataset.sometable" == tables[0].get_table_name() | ||
|
||
|
||
def test_one_part_table(): | ||
config = BigQueryV2Config() | ||
report = BigQueryV2Report() | ||
extractor = BigqueryLineageExtractor(config, report) | ||
|
||
ddl = "CREATE VIEW my_view as select * from sometable as a" | ||
view = BigqueryView( | ||
name="test", | ||
created=datetime.datetime.now(), | ||
last_altered=datetime.datetime.now(), | ||
comment="", | ||
ddl=ddl, | ||
) | ||
tables = extractor.parse_view_lineage("my_project", "my_dataset", view) | ||
assert 1 == len(tables) | ||
assert "my_project.my_dataset.sometable" == tables[0].get_table_name() | ||
|
||
|
||
def test_create_statement_with_multiple_table(): | ||
config = BigQueryV2Config() | ||
report = BigQueryV2Report() | ||
extractor = BigqueryLineageExtractor(config, report) | ||
|
||
ddl = "CREATE VIEW my_view as select * from my_project_2.my_dataset_2.sometable union select * from my_project_2.my_dataset_2.sometable2 as a" | ||
view = BigqueryView( | ||
name="test", | ||
created=datetime.datetime.now(), | ||
last_altered=datetime.datetime.now(), | ||
comment="", | ||
ddl=ddl, | ||
) | ||
tables = extractor.parse_view_lineage("my_project", "my_dataset", view) | ||
tables.sort(key=lambda e: e.get_table_name()) | ||
assert 2 == len(tables) | ||
assert "my_project_2.my_dataset_2.sometable" == tables[0].get_table_name() | ||
assert "my_project_2.my_dataset_2.sometable2" == tables[1].get_table_name() |