Skip to content

Commit

Permalink
change code
Browse files Browse the repository at this point in the history
  • Loading branch information
corsettigyg committed Aug 14, 2024
1 parent 436fb8c commit b6ae18e
Showing 1 changed file with 4 additions and 2 deletions.
6 changes: 4 additions & 2 deletions cosmos/dbt/project.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,16 +44,18 @@ def create_symlinks(project_path: Path, project_conn_id: str, tmp_dir: Path, ign
# this is linked to dbt deps so if dbt deps is true then ignore existing dbt_packages folder
ignore_paths.append("dbt_packages")
if project_conn_id:
from airflow.providers.amazon.aws.hooks.s3 import S3Hook
# Handle S3 path copying
s3_hook = S3Hook(aws_conn_id=project_conn_id)
bucket_name, key_prefix = project_path.parts[0], '/'.join(project_path.parts[1:])
print(bucket_name, key_prefix)

for obj in s3_hook.list_keys(bucket_name=bucket_name, prefix=key_prefix):
relative_path = Path(obj).relative_to(key_prefix)
local_path = tmp_dir / relative_path
local_path.parent.mkdir(parents=True, exist_ok=True)
# Download the file to the local path
s3_hook.download_file(bucket_name=bucket_name, key=obj, local_path=str(local_path))
else:
# Handle local symlinking
for child_name in project_path.iterdir():
if child_name.name not in ignore_paths:
os.symlink(project_path / child_name, tmp_dir / child_name.name)
Expand Down

0 comments on commit b6ae18e

Please sign in to comment.