diff --git a/cosmos/dbt/project.py b/cosmos/dbt/project.py index 8fa572092..d4e88ee28 100644 --- a/cosmos/dbt/project.py +++ b/cosmos/dbt/project.py @@ -44,16 +44,18 @@ def create_symlinks(project_path: Path, project_conn_id: str, tmp_dir: Path, ign # this is linked to dbt deps so if dbt deps is true then ignore existing dbt_packages folder ignore_paths.append("dbt_packages") if project_conn_id: - from airflow.providers.amazon.aws.hooks.s3 import S3Hook + # Handle S3 path copying s3_hook = S3Hook(aws_conn_id=project_conn_id) bucket_name, key_prefix = project_path.parts[0], '/'.join(project_path.parts[1:]) - print(bucket_name, key_prefix) + for obj in s3_hook.list_keys(bucket_name=bucket_name, prefix=key_prefix): relative_path = Path(obj).relative_to(key_prefix) local_path = tmp_dir / relative_path local_path.parent.mkdir(parents=True, exist_ok=True) + # Download the file to the local path s3_hook.download_file(bucket_name=bucket_name, key=obj, local_path=str(local_path)) else: + # Handle local symlinking for child_name in project_path.iterdir(): if child_name.name not in ignore_paths: os.symlink(project_path / child_name, tmp_dir / child_name.name)