diff --git a/metadata-ingestion/src/datahub/ingestion/source/delta_lake/config.py b/metadata-ingestion/src/datahub/ingestion/source/delta_lake/config.py index 28abad5d918bf..043facca1ebb6 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/delta_lake/config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/delta_lake/config.py @@ -59,6 +59,10 @@ class Config: default=AllowDenyPattern.allow_all(), description="regex patterns for tables to filter in ingestion.", ) + version_history_lookback: Optional[int] = Field( + default=1, + description="Number of previous version histories to be ingested. Defaults to 1. If set to -1 all version history will be ingested.", + ) s3: Optional[S3] = Field() @@ -72,14 +76,20 @@ def is_s3(self): def get_complete_path(self): return self._complete_path + @pydantic.validator("version_history_lookback") + def negative_version_history_implies_no_limit(cls, v): + if v and v < 0: + return None + return v + @pydantic.root_validator() def validate_config(cls, values: Dict) -> Dict[str, Any]: - values["_is_s3"] = is_s3_uri(values["base_path"]) + values["_is_s3"] = is_s3_uri(values.get("base_path", "")) if values["_is_s3"]: - if values["s3"] is None: + if values.get("s3") is None: raise ValueError("s3 config must be set for s3 path") - values["_complete_path"] = values["base_path"] - if values["relative_path"] is not None: + values["_complete_path"] = values.get("base_path") + if values.get("relative_path") is not None: values[ "_complete_path" ] = f"{values['_complete_path'].rstrip('/')}/{values['relative_path'].lstrip('/')}" diff --git a/metadata-ingestion/src/datahub/ingestion/source/delta_lake/delta_lake_utils.py b/metadata-ingestion/src/datahub/ingestion/source/delta_lake/delta_lake_utils.py index f1be3dc59c692..32f22b9d84773 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/delta_lake/delta_lake_utils.py +++ b/metadata-ingestion/src/datahub/ingestion/source/delta_lake/delta_lake_utils.py @@ -24,11 +24,7 @@ def read_delta_table( except PyDeltaTableError as e: if "Not a Delta table" not in str(e): - import pdb - - pdb.set_trace() raise e - return delta_table diff --git a/metadata-ingestion/src/datahub/ingestion/source/delta_lake/source.py b/metadata-ingestion/src/datahub/ingestion/source/delta_lake/source.py index 42837ac833b09..e32ba5d5c3bb3 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/delta_lake/source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/delta_lake/source.py @@ -1,5 +1,7 @@ import logging import os +import time +from datetime import datetime from typing import Callable, Iterable, List from deltalake import DeltaTable @@ -8,6 +10,7 @@ make_data_platform_urn, make_dataset_urn_with_platform_instance, ) +from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.ingestion.api.common import PipelineContext, WorkUnit from datahub.ingestion.api.decorators import ( SourceCapability, @@ -41,8 +44,11 @@ SchemaMetadata, ) from datahub.metadata.schema_classes import ( + ChangeTypeClass, DatasetPropertiesClass, NullTypeClass, + OperationClass, + OperationTypeClass, OtherSchemaClass, ) from datahub.telemetry import telemetry @@ -54,6 +60,19 @@ "platform", ] +OPERATION_STATEMENT_TYPES = { + "INSERT": OperationTypeClass.INSERT, + "UPDATE": OperationTypeClass.UPDATE, + "DELETE": OperationTypeClass.DELETE, + "MERGE": OperationTypeClass.UPDATE, + "CREATE": OperationTypeClass.CREATE, + "CREATE_TABLE_AS_SELECT": OperationTypeClass.CREATE, + "CREATE_SCHEMA": OperationTypeClass.CREATE, + "DROP_TABLE": OperationTypeClass.DROP, + "REPLACE TABLE AS SELECT": OperationTypeClass.UPDATE, + "COPY INTO": OperationTypeClass.UPDATE, +} + @platform_name("Delta Lake", id="delta-lake") @config_class(DeltaLakeSourceConfig) @@ -122,6 +141,58 @@ def get_fields(self, delta_table: DeltaTable) -> List[SchemaField]: return fields + def _create_operation_aspect_wu( + self, delta_table: DeltaTable, dataset_urn: str + ) -> Iterable[MetadataWorkUnit]: + for hist in delta_table.history( + limit=self.source_config.version_history_lookback + ): + + # History schema picked up from https://docs.delta.io/latest/delta-utility.html#retrieve-delta-table-history + reported_time: int = int(time.time() * 1000) + last_updated_timestamp: int = hist["timestamp"] + statement_type = OPERATION_STATEMENT_TYPES.get( + hist.get("operation"), OperationTypeClass.CUSTOM + ) + custom_type = ( + hist.get("operation") + if statement_type == OperationTypeClass.CUSTOM + else None + ) + + operation_custom_properties = dict() + for key, val in hist.items(): + if val is not None: + if isinstance(val, dict): + for k, v in val: + if v is not None: + operation_custom_properties[f"{key}_{k}"] = str(v) + else: + operation_custom_properties[key] = str(val) + operation_custom_properties.pop("timestamp", None) + operation_custom_properties.pop("operation", None) + operation_aspect = OperationClass( + timestampMillis=reported_time, + lastUpdatedTimestamp=last_updated_timestamp, + operationType=statement_type, + customOperationType=custom_type, + customProperties=operation_custom_properties, + ) + + mcp = MetadataChangeProposalWrapper( + entityType="dataset", + aspectName="operation", + changeType=ChangeTypeClass.UPSERT, + entityUrn=dataset_urn, + aspect=operation_aspect, + ) + operational_wu = MetadataWorkUnit( + id=f"{datetime.fromtimestamp(last_updated_timestamp / 1000).isoformat()}-operation-aspect-{dataset_urn}", + mcp=mcp, + ) + self.report.report_workunit(operational_wu) + yield operational_wu + def ingest_table( self, delta_table: DeltaTable, path: str ) -> Iterable[MetadataWorkUnit]: @@ -164,11 +235,6 @@ def ingest_table( "version": str(delta_table.version()), "location": self.source_config.get_complete_path(), } - customProperties.update(delta_table.history()[-1]) - customProperties["version_creation_time"] = customProperties["timestamp"] - del customProperties["timestamp"] - for key in customProperties.keys(): - customProperties[key] = str(customProperties[key]) dataset_properties = DatasetPropertiesClass( description=delta_table.metadata().description, @@ -221,6 +287,8 @@ def ingest_table( self.report.report_workunit(wu) yield wu + yield from self._create_operation_aspect_wu(delta_table, dataset_urn) + def process_folder( self, path: str, get_folders: Callable[[str], Iterable[str]] ) -> Iterable[MetadataWorkUnit]: diff --git a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_allow_table.json b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_allow_table.json index bdefaaac03e9a..53688610dd0ce 100644 --- a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_allow_table.json +++ b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_allow_table.json @@ -13,15 +13,7 @@ "table_creation_time": "1655831648843", "id": "de711767-c7b9-4c33-99d7-510978dc1fa5", "version": "4", - "location": "tests/integration/delta_lake/test_data/delta_tables", - "operation": "WRITE", - "operationParameters": "{}", - "readVersion": "3", - "isolationLevel": "Serializable", - "isBlindAppend": "True", - "operationMetrics": "{}", - "engineInfo": "local Delta-Standalone/0.4.0", - "version_creation_time": "1655831649788" + "location": "tests/integration/delta_lake/test_data/delta_tables" }, "externalUrl": null, "name": "my_table_no_name", @@ -514,6 +506,101 @@ "properties": null } }, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831649166}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"0\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831649715}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"1\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831649731}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"2\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831649754}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"3\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831649788}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, { "auditHeader": null, "proposedSnapshot": { @@ -528,14 +615,7 @@ "table_creation_time": "1655664813952", "id": "eca9d2a0-4ce6-4ace-a732-75fda0157fb8", "version": "0", - "location": "tests/integration/delta_lake/test_data/delta_tables", - "operation": "CONVERT", - "operationParameters": "{}", - "isolationLevel": "Serializable", - "isBlindAppend": "True", - "operationMetrics": "{}", - "engineInfo": "local Delta-Standalone/0.4.0", - "version_creation_time": "1655664815399" + "location": "tests/integration/delta_lake/test_data/delta_tables" }, "externalUrl": null, "name": "my_table", @@ -727,6 +807,25 @@ "properties": null } }, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/sales,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"CONVERT\", \"customProperties\": {\"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655664815399}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, { "auditHeader": null, "proposedSnapshot": { @@ -741,15 +840,7 @@ "table_creation_time": "1655831476360", "id": "628d06df-ecb0-4314-a97e-75d8872db7c3", "version": "4", - "location": "tests/integration/delta_lake/test_data/delta_tables", - "operation": "WRITE", - "operationParameters": "{}", - "readVersion": "3", - "isolationLevel": "Serializable", - "isBlindAppend": "True", - "operationMetrics": "{}", - "engineInfo": "local Delta-Standalone/0.4.0", - "version_creation_time": "1655831477768" + "location": "tests/integration/delta_lake/test_data/delta_tables" }, "externalUrl": null, "name": "test-table-basic", @@ -881,6 +972,101 @@ "properties": null } }, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831476907}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"0\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477701}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"1\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477726}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"2\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477745}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"3\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477768}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, { "auditHeader": null, "proposedSnapshot": { @@ -895,15 +1081,7 @@ "table_creation_time": "1655831864836", "id": "3775a0fd-4f58-4dea-b71a-e3fedb10f5b4", "version": "4", - "location": "tests/integration/delta_lake/test_data/delta_tables", - "operation": "WRITE", - "operationParameters": "{}", - "readVersion": "3", - "isolationLevel": "Serializable", - "isBlindAppend": "True", - "operationMetrics": "{}", - "engineInfo": "local Delta-Standalone/0.4.0", - "version_creation_time": "1655831866541" + "location": "tests/integration/delta_lake/test_data/delta_tables" }, "externalUrl": null, "name": "test-table-inner", @@ -1110,5 +1288,100 @@ "registryVersion": null, "properties": null } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831865396}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"0\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831866337}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"1\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831866398}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"2\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831866447}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"3\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831866541}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_inner_table.json b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_inner_table.json index 385e5efe01d4f..161d97f2d37f6 100644 --- a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_inner_table.json +++ b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_inner_table.json @@ -13,15 +13,7 @@ "table_creation_time": "1655831648843", "id": "de711767-c7b9-4c33-99d7-510978dc1fa5", "version": "4", - "location": "tests/integration/delta_lake/test_data/delta_tables", - "operation": "WRITE", - "operationParameters": "{}", - "readVersion": "3", - "isolationLevel": "Serializable", - "isBlindAppend": "True", - "operationMetrics": "{}", - "engineInfo": "local Delta-Standalone/0.4.0", - "version_creation_time": "1655831649788" + "location": "tests/integration/delta_lake/test_data/delta_tables" }, "externalUrl": null, "name": "my_table_no_name", @@ -514,6 +506,101 @@ "properties": null } }, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831649166}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"0\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831649715}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"1\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831649731}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"2\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831649754}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"3\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831649788}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, { "auditHeader": null, "proposedSnapshot": { @@ -528,14 +615,7 @@ "table_creation_time": "1655664813952", "id": "eca9d2a0-4ce6-4ace-a732-75fda0157fb8", "version": "0", - "location": "tests/integration/delta_lake/test_data/delta_tables", - "operation": "CONVERT", - "operationParameters": "{}", - "isolationLevel": "Serializable", - "isBlindAppend": "True", - "operationMetrics": "{}", - "engineInfo": "local Delta-Standalone/0.4.0", - "version_creation_time": "1655664815399" + "location": "tests/integration/delta_lake/test_data/delta_tables" }, "externalUrl": null, "name": "my_table", @@ -727,6 +807,25 @@ "properties": null } }, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/sales,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"CONVERT\", \"customProperties\": {\"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655664815399}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, { "auditHeader": null, "proposedSnapshot": { @@ -741,15 +840,7 @@ "table_creation_time": "1655831476360", "id": "628d06df-ecb0-4314-a97e-75d8872db7c3", "version": "4", - "location": "tests/integration/delta_lake/test_data/delta_tables", - "operation": "WRITE", - "operationParameters": "{}", - "readVersion": "3", - "isolationLevel": "Serializable", - "isBlindAppend": "True", - "operationMetrics": "{}", - "engineInfo": "local Delta-Standalone/0.4.0", - "version_creation_time": "1655831477768" + "location": "tests/integration/delta_lake/test_data/delta_tables" }, "externalUrl": null, "name": "test-table-basic", @@ -881,6 +972,101 @@ "properties": null } }, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831476907}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"0\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477701}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"1\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477726}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"2\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477745}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"3\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477768}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, { "auditHeader": null, "proposedSnapshot": { @@ -895,15 +1081,7 @@ "table_creation_time": "1655831864836", "id": "3775a0fd-4f58-4dea-b71a-e3fedb10f5b4", "version": "4", - "location": "tests/integration/delta_lake/test_data/delta_tables", - "operation": "WRITE", - "operationParameters": "{}", - "readVersion": "3", - "isolationLevel": "Serializable", - "isBlindAppend": "True", - "operationMetrics": "{}", - "engineInfo": "local Delta-Standalone/0.4.0", - "version_creation_time": "1655831866541" + "location": "tests/integration/delta_lake/test_data/delta_tables" }, "externalUrl": null, "name": "test-table-inner", @@ -1110,5 +1288,100 @@ "registryVersion": null, "properties": null } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831865396}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"0\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831866337}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"1\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831866398}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"2\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831866447}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"3\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831866541}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_relative_path.json b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_relative_path.json index 7ff5db60855c3..8e9cb143ea9dd 100644 --- a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_relative_path.json +++ b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_relative_path.json @@ -13,15 +13,7 @@ "table_creation_time": "1655831476360", "id": "628d06df-ecb0-4314-a97e-75d8872db7c3", "version": "4", - "location": "tests/integration/delta_lake/test_data/delta_tables/my_table_basic/", - "operation": "WRITE", - "operationParameters": "{}", - "readVersion": "3", - "isolationLevel": "Serializable", - "isBlindAppend": "True", - "operationMetrics": "{}", - "engineInfo": "local Delta-Standalone/0.4.0", - "version_creation_time": "1655831477768" + "location": "tests/integration/delta_lake/test_data/delta_tables/my_table_basic/" }, "externalUrl": null, "name": "test-table-basic", @@ -209,5 +201,100 @@ "registryVersion": null, "properties": null } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831476907}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "relative_path.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"0\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477701}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "relative_path.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"1\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477726}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "relative_path.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"2\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477745}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "relative_path.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"3\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477768}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "relative_path.json", + "registryName": null, + "registryVersion": null, + "properties": null + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_single_table.json b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_single_table.json index 3bde0bc9da303..d48cd3c64cc81 100644 --- a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_single_table.json +++ b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_single_table.json @@ -13,15 +13,7 @@ "table_creation_time": "1655831476360", "id": "628d06df-ecb0-4314-a97e-75d8872db7c3", "version": "4", - "location": "tests/integration/delta_lake/test_data/delta_tables/my_table_basic", - "operation": "WRITE", - "operationParameters": "{}", - "readVersion": "3", - "isolationLevel": "Serializable", - "isBlindAppend": "True", - "operationMetrics": "{}", - "engineInfo": "local Delta-Standalone/0.4.0", - "version_creation_time": "1655831477768" + "location": "tests/integration/delta_lake/test_data/delta_tables/my_table_basic" }, "externalUrl": null, "name": "test-table-basic", @@ -513,5 +505,24 @@ "registryVersion": null, "properties": null } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"operationType\": \"CUSTOM\", \"customOperationType\": \"WRITE\", \"customProperties\": {\"readVersion\": \"3\", \"isolationLevel\": \"Serializable\", \"isBlindAppend\": \"True\", \"engineInfo\": \"local Delta-Standalone/0.4.0\"}, \"lastUpdatedTimestamp\": 1655831477768}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json", + "registryName": null, + "registryVersion": null, + "properties": null + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/delta_lake/sources/local/allow_table.json b/metadata-ingestion/tests/integration/delta_lake/sources/local/allow_table.json index 1001eaadb3b9b..ec3d4ea2f5769 100644 --- a/metadata-ingestion/tests/integration/delta_lake/sources/local/allow_table.json +++ b/metadata-ingestion/tests/integration/delta_lake/sources/local/allow_table.json @@ -6,6 +6,7 @@ "platform_instance": "my-platform", "table_pattern": { "allow": ["s*"] - } + }, + "version_history_lookback": -1 } -} \ No newline at end of file +} diff --git a/metadata-ingestion/tests/integration/delta_lake/sources/local/inner_table.json b/metadata-ingestion/tests/integration/delta_lake/sources/local/inner_table.json index 8997605527b4c..c7475b2d6e728 100644 --- a/metadata-ingestion/tests/integration/delta_lake/sources/local/inner_table.json +++ b/metadata-ingestion/tests/integration/delta_lake/sources/local/inner_table.json @@ -2,6 +2,7 @@ "type": "delta-lake", "config": { "env": "UAT", - "base_path": "tests/integration/delta_lake/test_data/delta_tables" + "base_path": "tests/integration/delta_lake/test_data/delta_tables", + "version_history_lookback": -1 } -} \ No newline at end of file +} diff --git a/metadata-ingestion/tests/integration/delta_lake/sources/local/relative_path.json b/metadata-ingestion/tests/integration/delta_lake/sources/local/relative_path.json index a1734aea8d319..ffd736baa814b 100644 --- a/metadata-ingestion/tests/integration/delta_lake/sources/local/relative_path.json +++ b/metadata-ingestion/tests/integration/delta_lake/sources/local/relative_path.json @@ -3,6 +3,7 @@ "config": { "env": "UAT", "base_path": "tests/integration/delta_lake/test_data/", - "relative_path":"delta_tables/my_table_basic/" + "relative_path":"delta_tables/my_table_basic/", + "version_history_lookback": -1 } } diff --git a/metadata-ingestion/tests/integration/delta_lake/sources/local/single_table.json b/metadata-ingestion/tests/integration/delta_lake/sources/local/single_table.json index a12a7e71515a5..27fa25e09e099 100644 --- a/metadata-ingestion/tests/integration/delta_lake/sources/local/single_table.json +++ b/metadata-ingestion/tests/integration/delta_lake/sources/local/single_table.json @@ -4,4 +4,4 @@ "env": "UAT", "base_path": "tests/integration/delta_lake/test_data/delta_tables/my_table_basic" } -} \ No newline at end of file +}