Skip to content

Commit

Permalink
fix(ingest/dbt): only generate one subtype (#10615)
Browse files Browse the repository at this point in the history
  • Loading branch information
@hsheth2
hsheth2 authored May 30, 2024
1 parent d78287c commit db965d6
Show file tree
Hide file tree
Showing 9 changed files with 39 additions and 97 deletions.
7 changes: 0 additions & 7 deletions metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@
from datahub.ingestion.api.source import MetadataWorkUnitProcessor
from datahub.ingestion.api.source_helpers import auto_workunit
from datahub.ingestion.api.workunit import MetadataWorkUnit
from datahub.ingestion.source.common.subtypes import DatasetSubTypes
from datahub.ingestion.source.dbt.dbt_tests import (
DBTTest,
DBTTestResult,
Expand Down Expand Up @@ -1739,12 +1738,6 @@ def _create_subType_wu(
return None

subtypes: List[str] = [node.node_type.capitalize()]
if node.materialization == "table":
subtypes.append(DatasetSubTypes.TABLE)

if node.node_type == "model" or node.node_type == "snapshot":
# We need to add the view subtype so that the view properties tab shows up in the UI.
subtypes.append(DatasetSubTypes.VIEW)

return MetadataChangeProposalWrapper(
entityUrn=node_datahub_urn,
Expand Down
14 changes: 4 additions & 10 deletions metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,7 @@
"aspect": {
"json": {
"typeNames": [
"Model",
"View"
"Model"
]
}
},
Expand Down Expand Up @@ -372,9 +371,7 @@
"aspect": {
"json": {
"typeNames": [
"Model",
"Table",
"View"
"Model"
]
}
},
Expand Down Expand Up @@ -606,8 +603,7 @@
"aspect": {
"json": {
"typeNames": [
"Model",
"View"
"Model"
]
}
},
Expand Down Expand Up @@ -943,9 +939,7 @@
"aspect": {
"json": {
"typeNames": [
"Model",
"Table",
"View"
"Model"
]
}
},
Expand Down
17 changes: 5 additions & 12 deletions metadata-ingestion/tests/integration/dbt/dbt_test_column_meta_mapping_golden.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,7 @@
"aspect": {
"json": {
"typeNames": [
"Model",
"View"
"Model"
]
}
},
Expand Down Expand Up @@ -337,9 +336,7 @@
"aspect": {
"json": {
"typeNames": [
"Model",
"Table",
"View"
"Model"
]
}
},
Expand Down Expand Up @@ -559,8 +556,7 @@
"aspect": {
"json": {
"typeNames": [
"Model",
"View"
"Model"
]
}
},
Expand Down Expand Up @@ -888,9 +884,7 @@
"aspect": {
"json": {
"typeNames": [
"Model",
"Table",
"View"
"Model"
]
}
},
Expand Down Expand Up @@ -1107,8 +1101,7 @@
"aspect": {
"json": {
"typeNames": [
"Snapshot",
"View"
"Snapshot"
]
}
},
Expand Down
17 changes: 5 additions & 12 deletions metadata-ingestion/tests/integration/dbt/dbt_test_events_golden.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,7 @@
"aspect": {
"json": {
"typeNames": [
"Model",
"Table",
"View"
"Model"
]
}
},
Expand Down Expand Up @@ -233,9 +231,7 @@
"aspect": {
"json": {
"typeNames": [
"Model",
"Table",
"View"
"Model"
]
}
},
Expand Down Expand Up @@ -478,8 +474,7 @@
"aspect": {
"json": {
"typeNames": [
"Model",
"View"
"Model"
]
}
},
Expand Down Expand Up @@ -633,8 +628,7 @@
"aspect": {
"json": {
"typeNames": [
"Model",
"View"
"Model"
]
}
},
Expand Down Expand Up @@ -800,8 +794,7 @@
"aspect": {
"json": {
"typeNames": [
"Model",
"View"
"Model"
]
}
},
Expand Down
25 changes: 9 additions & 16 deletions metadata-ingestion/tests/integration/dbt/dbt_test_test_model_performance_golden.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,7 @@
"aspect": {
"json": {
"typeNames": [
"Model",
"View"
"Model"
]
}
},
Expand Down Expand Up @@ -337,9 +336,7 @@
"aspect": {
"json": {
"typeNames": [
"Model",
"Table",
"View"
"Model"
]
}
},
Expand Down Expand Up @@ -565,7 +562,7 @@
"name": "just-some-random-id_urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)",
"type": "BATCH_SCHEDULED",
"created": {
"time": 1643871600000,
"time": 1663355198240,
"actor": "urn:li:corpuser:datahub"
}
}
Expand Down Expand Up @@ -685,8 +682,7 @@
"aspect": {
"json": {
"typeNames": [
"Model",
"View"
"Model"
]
}
},
Expand Down Expand Up @@ -1020,7 +1016,7 @@
"name": "just-some-random-id_urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD)",
"type": "BATCH_SCHEDULED",
"created": {
"time": 1643871600000,
"time": 1663355198240,
"actor": "urn:li:corpuser:datahub"
}
}
Expand Down Expand Up @@ -1144,9 +1140,7 @@
"aspect": {
"json": {
"typeNames": [
"Model",
"Table",
"View"
"Model"
]
}
},
Expand Down Expand Up @@ -1349,7 +1343,7 @@
"name": "just-some-random-id_urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD)",
"type": "BATCH_SCHEDULED",
"created": {
"time": 1643871600000,
"time": 1663355198240,
"actor": "urn:li:corpuser:datahub"
}
}
Expand Down Expand Up @@ -1468,8 +1462,7 @@
"aspect": {
"json": {
"typeNames": [
"Snapshot",
"View"
"Snapshot"
]
}
},
Expand Down Expand Up @@ -1873,7 +1866,7 @@
"name": "just-some-random-id_urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD)",
"type": "BATCH_SCHEDULED",
"created": {
"time": 1643871600000,
"time": 1663355198240,
"actor": "urn:li:corpuser:datahub"
}
}
Expand Down
14 changes: 4 additions & 10 deletions ...ata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,7 @@
"aspect": {
"json": {
"typeNames": [
"Model",
"View"
"Model"
]
}
},
Expand Down Expand Up @@ -336,9 +335,7 @@
"aspect": {
"json": {
"typeNames": [
"Model",
"Table",
"View"
"Model"
]
}
},
Expand Down Expand Up @@ -552,8 +549,7 @@
"aspect": {
"json": {
"typeNames": [
"Model",
"View"
"Model"
]
}
},
Expand Down Expand Up @@ -889,9 +885,7 @@
"aspect": {
"json": {
"typeNames": [
"Model",
"Table",
"View"
"Model"
]
}
},
Expand Down
14 changes: 4 additions & 10 deletions ...ata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,7 @@
"aspect": {
"json": {
"typeNames": [
"Model",
"View"
"Model"
]
}
},
Expand Down Expand Up @@ -338,9 +337,7 @@
"aspect": {
"json": {
"typeNames": [
"Model",
"Table",
"View"
"Model"
]
}
},
Expand Down Expand Up @@ -555,8 +552,7 @@
"aspect": {
"json": {
"typeNames": [
"Model",
"View"
"Model"
]
}
},
Expand Down Expand Up @@ -893,9 +889,7 @@
"aspect": {
"json": {
"typeNames": [
"Model",
"Table",
"View"
"Model"
]
}
},
Expand Down
14 changes: 4 additions & 10 deletions ...ta-ingestion/tests/integration/dbt/dbt_test_with_non_incremental_lineage_mces_golden.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,7 @@
"aspect": {
"json": {
"typeNames": [
"Model",
"View"
"Model"
]
}
},
Expand Down Expand Up @@ -337,9 +336,7 @@
"aspect": {
"json": {
"typeNames": [
"Model",
"Table",
"View"
"Model"
]
}
},
Expand Down Expand Up @@ -553,8 +550,7 @@
"aspect": {
"json": {
"typeNames": [
"Model",
"View"
"Model"
]
}
},
Expand Down Expand Up @@ -890,9 +886,7 @@
"aspect": {
"json": {
"typeNames": [
"Model",
"Table",
"View"
"Model"
]
}
},
Expand Down
Loading

0 comments on commit db965d6

Please sign in to comment.