Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: bundles are stored in job_history in their original format #344

Merged
merged 7 commits into from
Jun 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions src/deadline/client/job_bundle/saver.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.

import json
import os
from typing import Any
from ._yaml import deadline_yaml_dump


def save_yaml_or_json_to_file(
bundle_dir: str,
filename: str,
file_type: str,
data: Any,
) -> None:
"""
Saves data as either a JSON or YAML file depending on the file_type provided. Useful for saving
job bundle data files which can be in either format. file_type should be either "YAML" or "JSON".
"""
with open(
os.path.join(bundle_dir, f"{filename}.{file_type.lower()}"), "w", encoding="utf8"
) as f:
if file_type == "YAML":
deadline_yaml_dump(data, f)
elif file_type == "JSON":
json.dump(data, f, indent=2)
else:
raise RuntimeError(f"Unexpected file type '{file_type}' in job bundle:\n{bundle_dir}")
34 changes: 16 additions & 18 deletions src/deadline/client/ui/job_bundle_submitter.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
from __future__ import annotations
import copy
import json
import os
from logging import getLogger
from typing import Any, Optional, Dict
Expand All @@ -14,13 +13,13 @@
)

from ..exceptions import DeadlineOperationError
from ..job_bundle import deadline_yaml_dump
from ..job_bundle.loader import (
parse_yaml_or_json_content,
read_yaml_or_json,
read_yaml_or_json_object,
validate_directory_symlink_containment,
)
from ..job_bundle.saver import save_yaml_or_json_to_file
from ..job_bundle.parameters import (
JobParameter,
apply_job_parameters,
Expand Down Expand Up @@ -99,20 +98,6 @@ def on_create_job_bundle_callback(
for step in template["steps"]:
step["hostRequirements"] = copy.deepcopy(host_requirements)

with open(
os.path.join(job_bundle_dir, f"template.{file_type.lower()}"), "w", encoding="utf8"
) as f:
if file_type == "YAML":
deadline_yaml_dump(template, f)
elif file_type == "JSON":
json.dump(template, f, indent=1)

if asset_references:
with open(
os.path.join(job_bundle_dir, "asset_references.yaml"), "w", encoding="utf8"
) as f:
deadline_yaml_dump(asset_references.to_dict(), f)

# First filter the queue parameters to exclude any from the job template,
# then extend it with the job template parameters.
job_parameter_names = {param["name"] for param in settings.parameters}
Expand All @@ -137,8 +122,21 @@ def on_create_job_bundle_callback(
AssetReferences(),
)

with open(os.path.join(job_bundle_dir, "parameter_values.yaml"), "w", encoding="utf8") as f:
deadline_yaml_dump({"parameterValues": parameters_values}, f)
save_yaml_or_json_to_file(
bundle_dir=job_bundle_dir, filename="template", file_type=file_type, data=template
)
save_yaml_or_json_to_file(
bundle_dir=job_bundle_dir,
filename="asset_references",
file_type=file_type,
data=asset_references.to_dict(),
)
save_yaml_or_json_to_file(
bundle_dir=job_bundle_dir,
filename="parameter_values",
file_type=file_type,
data={"parameterValues": parameters_values},
)

# Ensure the job bundle doesn't contain files that resolve outside of the bundle directory
validate_directory_symlink_containment(input_job_bundle_dir)
Expand Down
27 changes: 27 additions & 0 deletions test/unit/deadline_client/job_bundle/test_saver.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.

import os
from pathlib import Path
import tempfile
import pytest
from deadline.client.job_bundle.saver import save_yaml_or_json_to_file


def test_save_yaml_or_json_to_file():
with tempfile.TemporaryDirectory() as temp_dir:
save_yaml_or_json_to_file(
bundle_dir=temp_dir, filename="test", data={"test": "test"}, file_type="YAML"
)
assert Path(os.path.join(temp_dir, "test.yaml")).read_text() == "test: test\n"

with tempfile.TemporaryDirectory() as temp_dir:
save_yaml_or_json_to_file(
bundle_dir=temp_dir, filename="test", data={"test": "test"}, file_type="JSON"
)
assert Path(os.path.join(temp_dir, "test.json")).read_text() == '{\n "test": "test"\n}'

with tempfile.TemporaryDirectory() as temp_dir:
with pytest.raises(RuntimeError):
save_yaml_or_json_to_file(
bundle_dir=temp_dir, filename="test", data={"test": "test"}, file_type="NOT_VALID"
)
Loading