diff --git a/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_common.py b/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_common.py index a1b850b56677a4..ec7dbc8251b200 100644 --- a/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_common.py +++ b/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_common.py @@ -21,6 +21,9 @@ # See more below: # https://documentation.sas.com/doc/en/pgmsascdc/9.4_3.5/acreldb/n0ejgx4895bofnn14rlguktfx5r3.htm "teradata", + # For SQL server, the default collation rules mean that all identifiers (schema, table, column names) + # are case preserving but case insensitive. + "mssql", } DIALECTS_WITH_DEFAULT_UPPERCASE_COLS = { # In some dialects, column identifiers are effectively case insensitive @@ -28,6 +31,9 @@ # automatically lowercase unquoted identifiers. "snowflake", } +assert DIALECTS_WITH_DEFAULT_UPPERCASE_COLS.issubset( + DIALECTS_WITH_CASE_INSENSITIVE_COLS +) class QueryType(enum.Enum): diff --git a/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py b/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py index 4d0f9f7d8cf59d..9adb792a4be518 100644 --- a/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py +++ b/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py @@ -5,7 +5,7 @@ import logging import traceback from collections import defaultdict -from typing import Any, Dict, List, Optional, Set, Tuple, Union +from typing import Any, Dict, List, Optional, Set, Tuple, TypeVar, Union import pydantic.dataclasses import sqlglot @@ -873,6 +873,49 @@ def _translate_internal_column_lineage( ) +_StrOrNone = TypeVar("_StrOrNone", str, Optional[str]) + + +def _normalize_db_or_schema( + db_or_schema: _StrOrNone, + dialect: sqlglot.Dialect, +) -> _StrOrNone: + if db_or_schema is None: + return None + + # In snowflake, table identifiers must be uppercased to match sqlglot's behavior. + if is_dialect_instance(dialect, "snowflake"): + return db_or_schema.upper() + + # In mssql, table identifiers must be lowercased. + elif is_dialect_instance(dialect, "mssql"): + return db_or_schema.lower() + + return db_or_schema + + +def _simplify_select_into(statement: sqlglot.exp.Expression) -> sqlglot.exp.Expression: + """ + Check if the expression is a SELECT INTO statement. If so, converts it into a CTAS. + Other expressions are returned as-is. + """ + + if not (isinstance(statement, sqlglot.exp.Select) and statement.args.get("into")): + return statement + + # Convert from SELECT INTO + # to CREATE TABLE AS SELECT + into_expr: sqlglot.exp.Into = statement.args["into"].pop() + into_table = into_expr.this + + create = sqlglot.exp.Create( + this=into_table, + kind="TABLE", + expression=statement, + ) + return create + + def _sqlglot_lineage_inner( sql: sqlglot.exp.ExpOrStr, schema_resolver: SchemaResolverInterface, @@ -885,12 +928,9 @@ def _sqlglot_lineage_inner( else: dialect = get_dialect(default_dialect) - if is_dialect_instance(dialect, "snowflake"): - # in snowflake, table identifiers must be uppercased to match sqlglot's behavior. - if default_db: - default_db = default_db.upper() - if default_schema: - default_schema = default_schema.upper() + default_db = _normalize_db_or_schema(default_db, dialect) + default_schema = _normalize_db_or_schema(default_schema, dialect) + if is_dialect_instance(dialect, "redshift") and not default_schema: # On Redshift, there's no "USE SCHEMA " command. The default schema # is public, and "current schema" is the one at the front of the search path. @@ -918,6 +958,8 @@ def _sqlglot_lineage_inner( # original_statement.sql(pretty=True, dialect=dialect), # ) + statement = _simplify_select_into(statement) + # Make sure the tables are resolved with the default db / schema. # This only works for Unionable statements. For other types of statements, # we have to do it manually afterwards, but that's slightly lower accuracy diff --git a/metadata-ingestion/src/datahub/sql_parsing/sqlglot_utils.py b/metadata-ingestion/src/datahub/sql_parsing/sqlglot_utils.py index 5413161d0b7a3d..bd98557e08aacc 100644 --- a/metadata-ingestion/src/datahub/sql_parsing/sqlglot_utils.py +++ b/metadata-ingestion/src/datahub/sql_parsing/sqlglot_utils.py @@ -61,7 +61,7 @@ def is_dialect_instance( else: platforms = list(platforms) - dialects = [sqlglot.Dialect.get_or_raise(platform) for platform in platforms] + dialects = [get_dialect(platform) for platform in platforms] if any(isinstance(dialect, dialect_class.__class__) for dialect_class in dialects): return True diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json b/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json index 8e06ba9a80906b..5881ad5de0bcb8 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json @@ -1024,6 +1024,32 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,commopsdb.dbo.v_ps_cd_retention,PROD)", "type": "TRANSFORMED" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,commopsdb.dbo.v_ps_cd_retention,PROD),client_director)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,commopsdb.dbo.v_ps_cd_retention,PROD),month_wid)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV),cd_agent_key)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,commopsdb.dbo.v_ps_cd_retention,PROD),client_manager_closing_month)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,commopsdb.dbo.v_ps_cd_retention,PROD),month_wid)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV),agent_key)" + ], + "confidenceScore": 1.0 + } ] } }, diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json index 558548bfc7a698..4302c41140dc6c 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json @@ -113,11 +113,11 @@ "aspect": { "json": { "customProperties": { - "job_id": "ba144ff3-f6f8-4a61-a8a5-5cf1ed172738", + "job_id": "01afcab8-187c-459f-828e-727196a1832d", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-09-16 15:59:53.077000", - "date_modified": "2024-09-16 15:59:53.217000", + "date_created": "2024-11-21 21:01:26.550000", + "date_modified": "2024-11-21 21:01:26.690000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", @@ -154,6 +154,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", @@ -231,10 +247,15 @@ "entityType": "container", "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -245,17 +266,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", + "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -341,10 +357,15 @@ "entityType": "container", "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -355,17 +376,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820", + "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -451,10 +467,15 @@ "entityType": "container", "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -465,17 +486,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec", + "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -561,10 +577,15 @@ "entityType": "container", "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -575,17 +596,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8", + "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -671,10 +687,15 @@ "entityType": "container", "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -685,17 +706,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5", + "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -781,10 +797,15 @@ "entityType": "container", "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -795,17 +816,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f", + "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -891,10 +907,15 @@ "entityType": "container", "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -905,17 +926,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7", + "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -1001,10 +1017,15 @@ "entityType": "container", "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -1015,17 +1036,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47", + "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -1111,10 +1127,15 @@ "entityType": "container", "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -1125,17 +1146,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619", + "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -1217,22 +1233,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1", @@ -1386,6 +1386,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", @@ -1459,22 +1475,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", @@ -1943,15 +1943,13 @@ } }, { - "entityType": "dataFlow", - "entityUrn": "urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", "changeType": "UPSERT", - "aspectName": "dataFlowInfo", + "aspectName": "container", "aspect": { "json": { - "customProperties": {}, - "externalUrl": "", - "name": "DemoData.Foo.stored_procedures" + "container": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9" } }, "systemMetadata": { @@ -1961,9 +1959,188 @@ } }, { - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", - "changeType": "UPSERT", + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "view_definition": "CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;\n", + "is_view": "True" + }, + "name": "PersonsView", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "DemoData.Foo.PersonsView", + "platform": "urn:li:dataPlatform:mssql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "LastName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "FirstName", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Age", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;\n", + "viewLanguage": "SQL" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + }, + { + "id": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", + "urn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD)", + "changeType": "UPSERT", + "aspectName": "dataFlowInfo", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "", + "name": "DemoData.Foo.stored_procedures" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", + "changeType": "UPSERT", "aspectName": "dataJobInfo", "aspect": { "json": { @@ -1973,8 +2150,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-09-16 15:59:53.010000", - "date_modified": "2024-09-16 15:59:53.010000" + "date_created": "2024-11-21 21:01:26.483000", + "date_modified": "2024-11-21 21:01:26.483000" }, "externalUrl": "", "name": "DemoData.Foo.Proc.With.SpecialChar", @@ -2007,6 +2184,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", @@ -2084,10 +2277,15 @@ "entityType": "container", "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -2098,17 +2296,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", + "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -2194,10 +2387,15 @@ "entityType": "container", "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -2208,17 +2406,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f", + "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -2300,22 +2493,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d", @@ -2425,6 +2602,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:8b7691fec458d7383d5bc4e213831375", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:8b7691fec458d7383d5bc4e213831375", @@ -2502,10 +2695,15 @@ "entityType": "container", "entityUrn": "urn:li:container:8b7691fec458d7383d5bc4e213831375", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] } }, "systemMetadata": { @@ -2516,17 +2714,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:8b7691fec458d7383d5bc4e213831375", + "entityUrn": "urn:li:container:523d13eddd725607ec835a2459b05c9c", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", - "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - ] + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" } }, "systemMetadata": { @@ -2612,10 +2805,15 @@ "entityType": "container", "entityUrn": "urn:li:container:523d13eddd725607ec835a2459b05c9c", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] } }, "systemMetadata": { @@ -2626,17 +2824,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:523d13eddd725607ec835a2459b05c9c", + "entityUrn": "urn:li:container:29bd421b2225a415df9c750e77404c66", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", - "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - ] + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" } }, "systemMetadata": { @@ -2722,10 +2915,15 @@ "entityType": "container", "entityUrn": "urn:li:container:29bd421b2225a415df9c750e77404c66", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] } }, "systemMetadata": { @@ -2736,17 +2934,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:29bd421b2225a415df9c750e77404c66", + "entityUrn": "urn:li:container:a3c02df4bcc7280a89f539b793b04197", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", - "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - ] + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" } }, "systemMetadata": { @@ -2832,10 +3025,15 @@ "entityType": "container", "entityUrn": "urn:li:container:a3c02df4bcc7280a89f539b793b04197", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] } }, "systemMetadata": { @@ -2846,17 +3044,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:a3c02df4bcc7280a89f539b793b04197", + "entityUrn": "urn:li:container:c3b5d1cdc69a7d8faf0e1981e89b89d1", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", - "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - ] + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" } }, "systemMetadata": { @@ -2942,10 +3135,15 @@ "entityType": "container", "entityUrn": "urn:li:container:c3b5d1cdc69a7d8faf0e1981e89b89d1", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] } }, "systemMetadata": { @@ -2956,17 +3154,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:c3b5d1cdc69a7d8faf0e1981e89b89d1", + "entityUrn": "urn:li:container:2b937d85ae7545dc769766008a332f42", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", - "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - ] + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" } }, "systemMetadata": { @@ -3052,10 +3245,15 @@ "entityType": "container", "entityUrn": "urn:li:container:2b937d85ae7545dc769766008a332f42", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] } }, "systemMetadata": { @@ -3066,17 +3264,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:2b937d85ae7545dc769766008a332f42", + "entityUrn": "urn:li:container:a399d8bb765028abb9e55ae39846ca5e", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", - "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - ] + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" } }, "systemMetadata": { @@ -3162,10 +3355,15 @@ "entityType": "container", "entityUrn": "urn:li:container:a399d8bb765028abb9e55ae39846ca5e", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] } }, "systemMetadata": { @@ -3176,17 +3374,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:a399d8bb765028abb9e55ae39846ca5e", + "entityUrn": "urn:li:container:457efe38f0aec2af9ad681cf1b43b1cb", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", - "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - ] + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" } }, "systemMetadata": { @@ -3272,10 +3465,15 @@ "entityType": "container", "entityUrn": "urn:li:container:457efe38f0aec2af9ad681cf1b43b1cb", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] } }, "systemMetadata": { @@ -3286,17 +3484,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:457efe38f0aec2af9ad681cf1b43b1cb", + "entityUrn": "urn:li:container:1d87783ffe7e82210365dff4ca8ee7d1", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", - "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - ] + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" } }, "systemMetadata": { @@ -3382,10 +3575,15 @@ "entityType": "container", "entityUrn": "urn:li:container:1d87783ffe7e82210365dff4ca8ee7d1", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] } }, "systemMetadata": { @@ -3396,17 +3594,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:1d87783ffe7e82210365dff4ca8ee7d1", + "entityUrn": "urn:li:container:269d0067d130eda0399a534fc787054c", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", - "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - ] + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" } }, "systemMetadata": { @@ -3488,22 +3681,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:269d0067d130eda0399a534fc787054c", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:269d0067d130eda0399a534fc787054c", @@ -3669,6 +3846,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f721da08adde46586c0f113287cb60d1", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:f721da08adde46586c0f113287cb60d1", @@ -3742,22 +3935,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:f721da08adde46586c0f113287cb60d1", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:f721da08adde46586c0f113287cb60d1", @@ -4079,6 +4256,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f3cb304e29e178d0615ed5ee6aa4ad58", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:f3cb304e29e178d0615ed5ee6aa4ad58", @@ -4156,10 +4349,15 @@ "entityType": "container", "entityUrn": "urn:li:container:f3cb304e29e178d0615ed5ee6aa4ad58", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] } }, "systemMetadata": { @@ -4170,17 +4368,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:f3cb304e29e178d0615ed5ee6aa4ad58", + "entityUrn": "urn:li:container:752bb2abafeb2dae8f4adc7ffd547780", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", - "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - ] + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" } }, "systemMetadata": { @@ -4266,10 +4459,15 @@ "entityType": "container", "entityUrn": "urn:li:container:752bb2abafeb2dae8f4adc7ffd547780", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] } }, "systemMetadata": { @@ -4280,17 +4478,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:752bb2abafeb2dae8f4adc7ffd547780", + "entityUrn": "urn:li:container:46b713e3c7754c51649899f0f284ce34", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", - "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - ] + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" } }, "systemMetadata": { @@ -4376,10 +4569,15 @@ "entityType": "container", "entityUrn": "urn:li:container:46b713e3c7754c51649899f0f284ce34", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] } }, "systemMetadata": { @@ -4389,16 +4587,20 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:46b713e3c7754c51649899f0f284ce34", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "upstreamLineage", "aspect": { "json": { - "path": [ + "upstreams": [ { - "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", - "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)", + "type": "VIEW" } ] } @@ -4472,5 +4674,21 @@ "runId": "mssql-test", "lastRunId": "no-run-id-provided" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json index 161d40ea91d91a..0a50556edc6388 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json @@ -113,11 +113,11 @@ "aspect": { "json": { "customProperties": { - "job_id": "ba144ff3-f6f8-4a61-a8a5-5cf1ed172738", + "job_id": "01afcab8-187c-459f-828e-727196a1832d", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-09-16 15:59:53.077000", - "date_modified": "2024-09-16 15:59:53.217000", + "date_created": "2024-11-21 21:01:26.550000", + "date_modified": "2024-11-21 21:01:26.690000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", @@ -154,6 +154,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", @@ -231,10 +247,15 @@ "entityType": "container", "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -245,17 +266,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", + "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -341,10 +357,15 @@ "entityType": "container", "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -355,17 +376,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820", + "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -451,10 +467,15 @@ "entityType": "container", "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -465,17 +486,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec", + "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -561,10 +577,15 @@ "entityType": "container", "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -575,17 +596,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8", + "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -671,10 +687,15 @@ "entityType": "container", "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -685,17 +706,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5", + "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -781,10 +797,15 @@ "entityType": "container", "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -795,17 +816,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f", + "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -891,10 +907,15 @@ "entityType": "container", "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -905,17 +926,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7", + "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -1001,10 +1017,15 @@ "entityType": "container", "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -1015,17 +1036,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47", + "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -1111,10 +1127,15 @@ "entityType": "container", "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -1125,17 +1146,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619", + "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -1217,22 +1233,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1", @@ -1386,6 +1386,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", @@ -1459,22 +1475,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", @@ -1942,6 +1942,183 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "view_definition": "CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;\n", + "is_view": "True" + }, + "name": "PersonsView", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "DemoData.Foo.PersonsView", + "platform": "urn:li:dataPlatform:mssql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "LastName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "FirstName", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Age", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;\n", + "viewLanguage": "SQL" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + }, + { + "id": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", + "urn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataFlow", "entityUrn": "urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD)", @@ -1973,8 +2150,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-09-16 15:59:53.010000", - "date_modified": "2024-09-16 15:59:53.010000" + "date_created": "2024-11-21 21:01:26.483000", + "date_modified": "2024-11-21 21:01:26.483000" }, "externalUrl": "", "name": "DemoData.Foo.Proc.With.SpecialChar", @@ -2007,6 +2184,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", @@ -2084,10 +2277,15 @@ "entityType": "container", "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -2098,17 +2296,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", + "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -2194,10 +2387,15 @@ "entityType": "container", "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -2208,17 +2406,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f", + "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -2304,10 +2497,15 @@ "entityType": "container", "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -2317,16 +2515,20 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "upstreamLineage", "aspect": { "json": { - "path": [ + "upstreams": [ { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)", + "type": "VIEW" } ] } @@ -2400,5 +2602,21 @@ "runId": "mssql-test", "lastRunId": "no-run-id-provided" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json index 161d40ea91d91a..0a50556edc6388 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json @@ -113,11 +113,11 @@ "aspect": { "json": { "customProperties": { - "job_id": "ba144ff3-f6f8-4a61-a8a5-5cf1ed172738", + "job_id": "01afcab8-187c-459f-828e-727196a1832d", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-09-16 15:59:53.077000", - "date_modified": "2024-09-16 15:59:53.217000", + "date_created": "2024-11-21 21:01:26.550000", + "date_modified": "2024-11-21 21:01:26.690000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", @@ -154,6 +154,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", @@ -231,10 +247,15 @@ "entityType": "container", "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -245,17 +266,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", + "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -341,10 +357,15 @@ "entityType": "container", "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -355,17 +376,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820", + "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -451,10 +467,15 @@ "entityType": "container", "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -465,17 +486,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec", + "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -561,10 +577,15 @@ "entityType": "container", "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -575,17 +596,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8", + "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -671,10 +687,15 @@ "entityType": "container", "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -685,17 +706,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5", + "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -781,10 +797,15 @@ "entityType": "container", "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -795,17 +816,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f", + "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -891,10 +907,15 @@ "entityType": "container", "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -905,17 +926,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7", + "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -1001,10 +1017,15 @@ "entityType": "container", "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -1015,17 +1036,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47", + "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -1111,10 +1127,15 @@ "entityType": "container", "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -1125,17 +1146,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619", + "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -1217,22 +1233,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1", @@ -1386,6 +1386,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", @@ -1459,22 +1475,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", @@ -1942,6 +1942,183 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "view_definition": "CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;\n", + "is_view": "True" + }, + "name": "PersonsView", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "DemoData.Foo.PersonsView", + "platform": "urn:li:dataPlatform:mssql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "LastName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "FirstName", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Age", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;\n", + "viewLanguage": "SQL" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + }, + { + "id": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", + "urn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataFlow", "entityUrn": "urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD)", @@ -1973,8 +2150,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-09-16 15:59:53.010000", - "date_modified": "2024-09-16 15:59:53.010000" + "date_created": "2024-11-21 21:01:26.483000", + "date_modified": "2024-11-21 21:01:26.483000" }, "externalUrl": "", "name": "DemoData.Foo.Proc.With.SpecialChar", @@ -2007,6 +2184,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", @@ -2084,10 +2277,15 @@ "entityType": "container", "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -2098,17 +2296,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", + "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -2194,10 +2387,15 @@ "entityType": "container", "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -2208,17 +2406,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f", + "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -2304,10 +2497,15 @@ "entityType": "container", "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -2317,16 +2515,20 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "upstreamLineage", "aspect": { "json": { - "path": [ + "upstreams": [ { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)", + "type": "VIEW" } ] } @@ -2400,5 +2602,21 @@ "runId": "mssql-test", "lastRunId": "no-run-id-provided" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json index 29124f6fc156c4..0279a94084ce56 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json @@ -113,11 +113,11 @@ "aspect": { "json": { "customProperties": { - "job_id": "ba144ff3-f6f8-4a61-a8a5-5cf1ed172738", + "job_id": "01afcab8-187c-459f-828e-727196a1832d", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-09-16 15:59:53.077000", - "date_modified": "2024-09-16 15:59:53.217000", + "date_created": "2024-11-21 21:01:26.550000", + "date_modified": "2024-11-21 21:01:26.690000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", @@ -154,6 +154,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", @@ -231,10 +247,15 @@ "entityType": "container", "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -245,17 +266,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", + "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -341,10 +357,15 @@ "entityType": "container", "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -355,17 +376,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820", + "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -451,10 +467,15 @@ "entityType": "container", "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -465,17 +486,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec", + "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -561,10 +577,15 @@ "entityType": "container", "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -575,17 +596,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8", + "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -671,10 +687,15 @@ "entityType": "container", "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -685,17 +706,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5", + "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -781,10 +797,15 @@ "entityType": "container", "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -795,17 +816,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f", + "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -891,10 +907,15 @@ "entityType": "container", "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -905,17 +926,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7", + "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -1001,10 +1017,15 @@ "entityType": "container", "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -1015,17 +1036,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47", + "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -1111,10 +1127,15 @@ "entityType": "container", "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -1125,17 +1146,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619", + "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -1217,22 +1233,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1", @@ -1386,6 +1386,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", @@ -1459,22 +1475,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", @@ -1942,6 +1942,183 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "view_definition": "CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;\n", + "is_view": "True" + }, + "name": "PersonsView", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "demodata.foo.personsview", + "platform": "urn:li:dataPlatform:mssql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "LastName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "FirstName", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Age", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;\n", + "viewLanguage": "SQL" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + }, + { + "id": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", + "urn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataFlow", "entityUrn": "urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD)", @@ -1973,8 +2150,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-09-16 15:59:53.010000", - "date_modified": "2024-09-16 15:59:53.010000" + "date_created": "2024-11-21 21:01:26.483000", + "date_modified": "2024-11-21 21:01:26.483000" }, "externalUrl": "", "name": "DemoData.Foo.Proc.With.SpecialChar", @@ -2007,6 +2184,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", @@ -2084,10 +2277,15 @@ "entityType": "container", "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -2098,17 +2296,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", + "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -2194,10 +2387,15 @@ "entityType": "container", "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -2208,17 +2406,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f", + "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -2304,10 +2497,15 @@ "entityType": "container", "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -2317,16 +2515,66 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "upstreamLineage", "aspect": { "json": { - "path": [ + "upstreams": [ { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)", + "type": "VIEW" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),Age)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD),Age)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),FirstName)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD),FirstName)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD),ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),LastName)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD),LastName)" + ], + "confidenceScore": 1.0 } ] } diff --git a/metadata-ingestion/tests/integration/sql_server/setup/setup.sql b/metadata-ingestion/tests/integration/sql_server/setup/setup.sql index 77ecabc5a3fffc..f495db3b91cfae 100644 --- a/metadata-ingestion/tests/integration/sql_server/setup/setup.sql +++ b/metadata-ingestion/tests/integration/sql_server/setup/setup.sql @@ -32,6 +32,8 @@ CREATE TABLE Foo.Persons ( Age int ); GO +CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons; +GO CREATE TABLE Foo.SalesReason ( TempID int NOT NULL, diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_mssql_casing_resolver.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_mssql_casing_resolver.json new file mode 100644 index 00000000000000..072260f5cd1651 --- /dev/null +++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_mssql_casing_resolver.json @@ -0,0 +1,82 @@ +{ + "query_type": "SELECT", + "query_type_props": {}, + "query_fingerprint": "6a779a57ffb2598c301606d3a7d82142a7af8b102efa55a2c9a4e960fd55ac07", + "in_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.persons,PROD)" + ], + "out_tables": [], + "column_lineage": [ + { + "downstream": { + "table": null, + "column": "age", + "column_type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "native_column_type": "INTEGER" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.persons,PROD)", + "column": "Age" + } + ] + }, + { + "downstream": { + "table": null, + "column": "name", + "column_type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "native_column_type": "VARCHAR(16777216)" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.persons,PROD)", + "column": "Name" + } + ] + }, + { + "downstream": { + "table": null, + "column": "uppercased_col", + "column_type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "native_column_type": "VARCHAR(16777216)" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.persons,PROD)", + "column": "Uppercased_Col" + } + ] + }, + { + "downstream": { + "table": null, + "column": "count", + "column_type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "native_column_type": "INTEGER" + }, + "upstreams": [] + } + ], + "debug_info": { + "confidence": 0.9, + "generalized_statement": "SELECT Age, name, UPPERCASED_COL, COUNT(*) AS Count FROM Foo.Persons GROUP BY Age" + } +} \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_mssql_select_into.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_mssql_select_into.json new file mode 100644 index 00000000000000..ff8609922a9bd8 --- /dev/null +++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_mssql_select_into.json @@ -0,0 +1,48 @@ +{ + "query_type": "SELECT", + "query_type_props": {}, + "query_fingerprint": "d18c43ed4d7f0e303b88c40ac078c5505be473edf0136e42a77dccc090d1eb8b", + "in_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.persons,PROD)" + ], + "out_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.age_dist,PROD)" + ], + "column_lineage": [ + { + "downstream": { + "table": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.age_dist,PROD)", + "column": "AGE", + "column_type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "native_column_type": "INTEGER" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.persons,PROD)", + "column": "Age" + } + ] + }, + { + "downstream": { + "table": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.age_dist,PROD)", + "column": "Count", + "column_type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "native_column_type": "INTEGER" + }, + "upstreams": [] + } + ], + "debug_info": { + "confidence": 0.9, + "generalized_statement": "SELECT age AS AGE, COUNT(*) AS Count INTO Foo.age_dist FROM Foo.Persons GROUP BY Age" + } +} \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py index 170341230205f3..2df6f6fa26cb56 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py +++ b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py @@ -1279,3 +1279,47 @@ def test_sqlite_attach_database() -> None: expected_file=RESOURCE_DIR / "test_sqlite_attach_database.json", allow_table_error=True, ) + + +def test_mssql_casing_resolver() -> None: + assert_sql_result( + """\ +SELECT Age, name, UPPERCASED_COL, COUNT(*) as Count +FROM Foo.Persons +GROUP BY Age +""", + dialect="mssql", + default_db="NewData", + schemas={ + "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.persons,PROD)": { + "Age": "INTEGER", + "Name": "VARCHAR(16777216)", + "Uppercased_Col": "VARCHAR(16777216)", + }, + }, + expected_file=RESOURCE_DIR / "test_mssql_casing_resolver.json", + ) + + +def test_mssql_select_into() -> None: + assert_sql_result( + """\ +SELECT age as AGE, COUNT(*) as Count +INTO Foo.age_dist +FROM Foo.Persons +GROUP BY Age +""", + dialect="mssql", + default_db="NewData", + schemas={ + "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.persons,PROD)": { + "Age": "INTEGER", + "Name": "VARCHAR(16777216)", + }, + "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.age_dist,PROD)": { + "AGE": "INTEGER", + "Count": "INTEGER", + }, + }, + expected_file=RESOURCE_DIR / "test_mssql_select_into.json", + )