From c2d70c12d884230024cbb893f55eb6f1d0ff5043 Mon Sep 17 00:00:00 2001 From: Gahyun Suh <132245153+gahyusuh@users.noreply.github.com> Date: Sat, 6 Apr 2024 23:48:09 +0000 Subject: [PATCH] fix(job_attachments): handle case-insensitive path extraction on Windows Signed-off-by: Gahyun Suh <132245153+gahyusuh@users.noreply.github.com> --- src/deadline/job_attachments/upload.py | 29 +++++++-- .../deadline_job_attachments/test_upload.py | 64 ++++++++++++++++++- 2 files changed, 88 insertions(+), 5 deletions(-) diff --git a/src/deadline/job_attachments/upload.py b/src/deadline/job_attachments/upload.py index 954a0ddd..1ca9b1f4 100644 --- a/src/deadline/job_attachments/upload.py +++ b/src/deadline/job_attachments/upload.py @@ -832,7 +832,8 @@ def _get_asset_groups( abs_path=abs_path, local_type_locations=local_type_locations, ) - groupings[matched_root].inputs.add(abs_path) + matched_group = self._get_matched_group(matched_root, groupings) + matched_group.inputs.add(abs_path) for _path in output_paths: abs_path = Path(os.path.normpath(Path(_path).absolute())) @@ -849,7 +850,8 @@ def _get_asset_groups( abs_path=abs_path, local_type_locations=local_type_locations, ) - groupings[matched_root].outputs.add(abs_path) + matched_group = self._get_matched_group(matched_root, groupings) + matched_group.outputs.add(abs_path) for _path in referenced_paths: abs_path = Path(os.path.normpath(Path(_path).absolute())) @@ -865,7 +867,8 @@ def _get_asset_groups( abs_path=abs_path, local_type_locations=local_type_locations, ) - groupings[matched_root].references.add(abs_path) + matched_group = self._get_matched_group(matched_root, groupings) + matched_group.references.add(abs_path) # Finally, build the list of asset root groups for asset_group in groupings.values(): @@ -880,6 +883,20 @@ def _get_asset_groups( return list(groupings.values()) + def _get_matched_group( + self, root_path: str, groupings: dict[str, AssetRootGroup] + ) -> AssetRootGroup: + root_normcase = os.path.normcase(root_path) + matched_group = next( + (group for key, group in groupings.items() if os.path.normcase(key) == root_normcase), + None, + ) + if matched_group is None: + raise ValueError( + f"No group found for the root path '{root_path}' in the groupings dictionary: {groupings}" + ) + return matched_group + def _find_matched_root_from_local_type_locations( self, groupings: dict[str, AssetRootGroup], @@ -906,9 +923,13 @@ def _find_matched_root_from_local_type_locations( ) return matched_root else: + keys_normcase = [os.path.normcase(key) for key in groupings.keys()] top_directory = PurePath(abs_path).parts[0] - if top_directory not in groupings: + top_directory_normcase = os.path.normcase(top_directory) + if top_directory_normcase not in keys_normcase: groupings[top_directory] = AssetRootGroup() + else: + return top_directory_normcase return top_directory def _get_total_size_of_files(self, paths: list[str]) -> int: diff --git a/test/unit/deadline_job_attachments/test_upload.py b/test/unit/deadline_job_attachments/test_upload.py index 89d48836..c5c90dd8 100644 --- a/test/unit/deadline_job_attachments/test_upload.py +++ b/test/unit/deadline_job_attachments/test_upload.py @@ -1980,6 +1980,28 @@ def test_get_file_system_locations_by_type( {}, # File System Location (SHARED type) [], ), + ( + { + "/home/username/DOCS/inputs/input1.txt", + "/HOME/username/DOCS/inputs/input2.txt", + }, # input paths + {"/home/username/docs/outputs"}, # output paths + set(), # referenced paths + {}, # File System Location (LOCAL type) + {}, # File System Location (SHARED type) + [ + AssetRootGroup( + root_path="/", + inputs={ + Path("/home/username/DOCS/inputs/input1.txt"), + Path("/HOME/username/DOCS/inputs/input2.txt"), + }, + outputs={ + Path("/home/username/docs/outputs"), + }, + ), + ], + ), ( {"/home/username/docs/inputs/input1.txt"}, # input paths {"/home/username/docs/outputs"}, # output paths @@ -2122,6 +2144,42 @@ def test_get_asset_groups( {}, # File System Location (SHARED type) [], ), + ( + {"d:\\USERNAME\\DOCS\\inputs\\input1.txt"}, # input paths + {"D:\\username\\docs\\outputs"}, # output paths + set(), # referenced paths + {}, # File System Location (LOCAL type) + {}, # File System Location (SHARED type) + [ + AssetRootGroup( + root_path="D:\\username\\docs", + inputs={ + Path("d:\\USERNAME\\DOCS\\inputs\\input1.txt"), + }, + outputs={ + Path("D:\\username\\docs\\outputs"), + }, + ), + ], + ), + ( + {"D:\\username\\docs\\inputs\\input1.txt"}, # input paths + {"d:\\USERNAME\\DOCS\\outputs"}, # output paths + set(), # referenced paths + {}, # File System Location (LOCAL type) + {}, # File System Location (SHARED type) + [ + AssetRootGroup( + root_path="D:\\username\\docs", + inputs={ + Path("D:\\username\\docs\\inputs\\input1.txt"), + }, + outputs={ + Path("d:\\USERNAME\\DOCS\\outputs"), + }, + ), + ], + ), ( {"C:\\username\\docs\\inputs\\input1.txt"}, # input paths {"C:\\username\\docs\\outputs"}, # output paths @@ -2250,7 +2308,11 @@ def test_get_asset_groups_for_windows( sorted_result = sorted(result, key=lambda x: x.root_path) sorted_expected_result = sorted(expected_result, key=lambda x: x.root_path) - assert sorted_result == sorted_expected_result + assert len(sorted_result) == len(sorted_expected_result) + for i in range(len(sorted_result)): + assert sorted_result[i].root_path.upper() == sorted_expected_result[i].root_path.upper() + assert sorted_result[i].inputs == sorted_expected_result[i].inputs + assert sorted_result[i].outputs == sorted_expected_result[i].outputs @pytest.mark.skipif( sys.platform != "win32",