Skip to content

Commit

Permalink
fix(job_attachments): handle case-insensitive path extraction on Windows
Browse files Browse the repository at this point in the history 
Signed-off-by: Gahyun Suh <[email protected]>
  • Loading branch information
@gahyusuh
gahyusuh committed Apr 7, 2024
1 parent cfbff76 commit c2d70c1
Show file tree
Hide file tree
Showing 2 changed files with 88 additions and 5 deletions.
29 changes: 25 additions & 4 deletions src/deadline/job_attachments/upload.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -832,7 +832,8 @@ def _get_asset_groups(
abs_path=abs_path,
local_type_locations=local_type_locations,
)
groupings[matched_root].inputs.add(abs_path)
matched_group = self._get_matched_group(matched_root, groupings)
matched_group.inputs.add(abs_path)

for _path in output_paths:
abs_path = Path(os.path.normpath(Path(_path).absolute()))
Expand All @@ -849,7 +850,8 @@ def _get_asset_groups(
abs_path=abs_path,
local_type_locations=local_type_locations,
)
groupings[matched_root].outputs.add(abs_path)
matched_group = self._get_matched_group(matched_root, groupings)
matched_group.outputs.add(abs_path)

for _path in referenced_paths:
abs_path = Path(os.path.normpath(Path(_path).absolute()))
Expand All @@ -865,7 +867,8 @@ def _get_asset_groups(
abs_path=abs_path,
local_type_locations=local_type_locations,
)
groupings[matched_root].references.add(abs_path)
matched_group = self._get_matched_group(matched_root, groupings)
matched_group.references.add(abs_path)

# Finally, build the list of asset root groups
for asset_group in groupings.values():
Expand All @@ -880,6 +883,20 @@ def _get_asset_groups(

return list(groupings.values())

def _get_matched_group(
self, root_path: str, groupings: dict[str, AssetRootGroup]
) -> AssetRootGroup:
root_normcase = os.path.normcase(root_path)
matched_group = next(
(group for key, group in groupings.items() if os.path.normcase(key) == root_normcase),
None,
)
if matched_group is None:
raise ValueError(
f"No group found for the root path '{root_path}' in the groupings dictionary: {groupings}"
)
return matched_group

def _find_matched_root_from_local_type_locations(
self,
groupings: dict[str, AssetRootGroup],
Expand All @@ -906,9 +923,13 @@ def _find_matched_root_from_local_type_locations(
)
return matched_root
else:
keys_normcase = [os.path.normcase(key) for key in groupings.keys()]
top_directory = PurePath(abs_path).parts[0]
if top_directory not in groupings:
top_directory_normcase = os.path.normcase(top_directory)
if top_directory_normcase not in keys_normcase:
groupings[top_directory] = AssetRootGroup()
else:
return top_directory_normcase
return top_directory

def _get_total_size_of_files(self, paths: list[str]) -> int:
Expand Down
64 changes: 63 additions & 1 deletion test/unit/deadline_job_attachments/test_upload.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1980,6 +1980,28 @@ def test_get_file_system_locations_by_type(
{}, # File System Location (SHARED type)
[],
),
(
{
"/home/username/DOCS/inputs/input1.txt",
"/HOME/username/DOCS/inputs/input2.txt",
}, # input paths
{"/home/username/docs/outputs"}, # output paths
set(), # referenced paths
{}, # File System Location (LOCAL type)
{}, # File System Location (SHARED type)
[
AssetRootGroup(
root_path="/",
inputs={
Path("/home/username/DOCS/inputs/input1.txt"),
Path("/HOME/username/DOCS/inputs/input2.txt"),
},
outputs={
Path("/home/username/docs/outputs"),
},
),
],
),
(
{"/home/username/docs/inputs/input1.txt"}, # input paths
{"/home/username/docs/outputs"}, # output paths
Expand Down Expand Up @@ -2122,6 +2144,42 @@ def test_get_asset_groups(
{}, # File System Location (SHARED type)
[],
),
(
{"d:\\USERNAME\\DOCS\\inputs\\input1.txt"}, # input paths
{"D:\\username\\docs\\outputs"}, # output paths
set(), # referenced paths
{}, # File System Location (LOCAL type)
{}, # File System Location (SHARED type)
[
AssetRootGroup(
root_path="D:\\username\\docs",
inputs={
Path("d:\\USERNAME\\DOCS\\inputs\\input1.txt"),
},
outputs={
Path("D:\\username\\docs\\outputs"),
},
),
],
),
(
{"D:\\username\\docs\\inputs\\input1.txt"}, # input paths
{"d:\\USERNAME\\DOCS\\outputs"}, # output paths
set(), # referenced paths
{}, # File System Location (LOCAL type)
{}, # File System Location (SHARED type)
[
AssetRootGroup(
root_path="D:\\username\\docs",
inputs={
Path("D:\\username\\docs\\inputs\\input1.txt"),
},
outputs={
Path("d:\\USERNAME\\DOCS\\outputs"),
},
),
],
),
(
{"C:\\username\\docs\\inputs\\input1.txt"}, # input paths
{"C:\\username\\docs\\outputs"}, # output paths
Expand Down Expand Up @@ -2250,7 +2308,11 @@ def test_get_asset_groups_for_windows(
sorted_result = sorted(result, key=lambda x: x.root_path)
sorted_expected_result = sorted(expected_result, key=lambda x: x.root_path)

assert sorted_result == sorted_expected_result
assert len(sorted_result) == len(sorted_expected_result)
for i in range(len(sorted_result)):
assert sorted_result[i].root_path.upper() == sorted_expected_result[i].root_path.upper()
assert sorted_result[i].inputs == sorted_expected_result[i].inputs
assert sorted_result[i].outputs == sorted_expected_result[i].outputs

@pytest.mark.skipif(
sys.platform != "win32",
Expand Down

0 comments on commit c2d70c1

Please sign in to comment.