From 887b98516e486c7b38607fba22bc3910919012e8 Mon Sep 17 00:00:00 2001 From: Pang Wu Date: Sat, 13 Apr 2024 22:53:48 -0700 Subject: [PATCH] deltalake _delta_lake.py: Allow Glue catalog cross account access --- daft/io/_delta_lake.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/daft/io/_delta_lake.py b/daft/io/_delta_lake.py index 6bf8e764f4..814e36082a 100644 --- a/daft/io/_delta_lake.py +++ b/daft/io/_delta_lake.py @@ -72,7 +72,13 @@ def read_delta_lake( aws_secret_access_key=s3_config.access_key, aws_session_token=s3_config.session_token, ) - glue_table = glue.get_table(DatabaseName=table.database_name, Name=table.table_name) + if table.catalog_id is not None: + # Allow cross account access, table.catalog_id should be the target account id + glue_table = glue.get_table(CatalogId=table.catalog_id, DatabaseName=table.database_name, + Name=table.table_name) + else: + glue_table = glue.get_table(DatabaseName=table.database_name, Name=table.table_name) + # TODO(Clark): Fetch more than just the table URI from Glue Data Catalog. table_uri = glue_table["Table"]["StorageDescriptor"]["Location"] elif table.catalog == DataCatalogType.UNITY: