diff --git a/dags/roger/core/storage.py b/dags/roger/core/storage.py index c1a3e80..e3c9697 100644 --- a/dags/roger/core/storage.py +++ b/dags/roger/core/storage.py @@ -167,6 +167,9 @@ def merge_path(name, path: Path=None): if not os.path.exists(ROGER_DATA_DIR / 'merge'): os.makedirs(ROGER_DATA_DIR / 'merge') return str(ROGER_DATA_DIR / 'merge' / name) + if not os.path.exists(path): + os.makedirs(path) + return str(path.joinpath(name)) def merged_objects(file_type, path=None): diff --git a/dags/roger/tasks.py b/dags/roger/tasks.py index f545114..678f713 100755 --- a/dags/roger/tasks.py +++ b/dags/roger/tasks.py @@ -176,7 +176,8 @@ def avalon_commit_callback(context: DagContext, **kwargs): branch=temp_branch_name, repo=repo, # @TODO figure out how to pass real commit id here - commit_id=branch + commit_id=branch, + source_branch_name=branch ) # see what changes are going to be pushed from this branch to main branch @@ -205,6 +206,7 @@ def avalon_commit_callback(context: DagContext, **kwargs): repository=repo, branch=temp_branch_name ) + logger.info(f"deleted temp branch {temp_branch_name}") logger.info(f"deleting local dir {local_path}") files_to_clean = glob.glob(local_path + '**', recursive=True) + [local_path] diff --git a/requirements.txt b/requirements.txt index 3a0ee22..2c9dbbf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,11 +4,12 @@ elasticsearch==8.5.2 flatten-dict jsonpickle redisgraph-bulk-loader==0.12.3 +setuptools>=66 pytest PyYAML -git+https://github.com/helxplatform/dug@2.13.2 +git+https://github.com/helxplatform/dug@2.13.4 orjson kg-utils==0.0.6 bmt==1.1.0 -git+https://github.com/helxplatform/avalon.git@v1.0.1 +git+https://github.com/helxplatform/avalon.git@v1.1.0 linkml-runtime==1.6.0