feat: adds --storage-profile-id option to the bundle submit command (a…

…ws-deadline#442)

Problem:

Customers have expressed the need to be able to provide an override for
the storage profile id when submitting jobs using `deadline bundle submit`,
but that command doesn't provide a way to override that setting. The use-case
is submitting many jobs, in parallel, from the same host where jobs need
different storage profile ids.

Solution:

Add an override. Using the override does not store the given storage profile
id to the user's configuration file.

Signed-off-by: Daniel Neilson <[email protected]>

src/deadline/client/cli/_common.py

@@ -96,6 +96,12 @@ def _apply_cli_options_to_config(
         if queue_id:
             config_file.set_setting("defaults.queue_id", queue_id, config=config)
 
+        storage_profile_id = args.pop("storage_profile_id", None)
+        if storage_profile_id:
+            config_file.set_setting(
+                "settings.storage_profile_id", storage_profile_id, config=config
+            )
+
         job_id = args.pop("job_id", None)
         if job_id:
             config_file.set_setting("defaults.job_id", job_id, config=config)
@@ -111,7 +117,7 @@ def _apply_cli_options_to_config(
             )
     else:
         # Remove the standard option names from the args list
-        for name in ["profile", "farm_id", "queue_id", "job_id"]:
+        for name in ["profile", "farm_id", "queue_id", "job_id", "storage_profile_id"]:
             args.pop(name, None)
 
     # Check that the required options have values

src/deadline/client/cli/_groups/bundle_group.py

@@ -82,6 +82,7 @@ def validate_parameters(ctx, param, value):
 @click.option("--profile", help="The AWS profile to use.")
 @click.option("--farm-id", help="The farm to use.")
 @click.option("--queue-id", help="The queue to use.")
+@click.option("--storage-profile-id", help="The storage profile to use.")
 @click.option("--name", help="The job name to use in place of the one in the job bundle.")
 @click.option(
     "--priority",
@@ -140,7 +141,7 @@ def bundle_submit(
     """
     Submits an Open Job Description job bundle.
     """
-    # Get a temporary config object with the standard options handled
+    # Apply the CLI args to the config
     config = _apply_cli_options_to_config(required_options={"farm_id", "queue_id"}, **args)
 
     hash_callback_manager = _ProgressBarCallbackManager(length=100, label="Hashing Attachments")
@@ -226,6 +227,7 @@ def _decide_cancel_submission(upload_group: AssetUploadGroup) -> bool:
             and args.get("profile") is None
             and args.get("farm_id") is None
             and args.get("queue_id") is None
+            and args.get("storage_profile_id") is None
         ):
             set_setting("defaults.job_id", job_id)
 

test/unit/conftest.py

@@ -34,6 +34,11 @@ def fresh_deadline_config():
         # Yield the temp file name with it patched in as the
         # AWS Deadline Cloud config file
         with patch.object(config_file, "CONFIG_FILE_PATH", str(temp_file_path)):
+            # Write a telemetry id to force it getting saved to the config file. If we don't, then
+            # an ID will get generated and force a save of the config file in the middle of a test.
+            # Writing the config file may be undesirable in the middle of a test.
+            config_file.set_setting("telemetry.identifier", "00000000-0000-0000-0000-000000000000")
+
             yield str(temp_file_path)
     finally:
         temp_dir.cleanup()

test/unit/deadline_client/cli/test_cli_bundle.py

@@ -15,6 +15,7 @@
 
 from deadline.client import config
 from deadline.client.api import _queue_parameters
+import deadline.client.api as api_module
 from deadline.client.cli import main
 from deadline.client.cli._groups import bundle_group
 from deadline.client.api import _submit_job_bundle
@@ -290,6 +291,54 @@ def test_cli_bundle_job_name(fresh_deadline_config):
     assert result.exit_code == 0
 
 
+def test_cli_bundle_storage_profile_id(fresh_deadline_config):
+    """
+    Confirm that --storage-profile-id sets the ID that the job is submitted with, but does not
+    change the value of storage profile saved to the configuration file.
+    """
+    PRE_STORAGE_PROFILE_ID = "sp-11223344556677889900abbccddeeff"
+    CLI_STORAGE_PROFILE_ID = "sp-0000000000000000000000000000000"
+
+    config.set_setting("defaults.farm_id", MOCK_FARM_ID)
+    config.set_setting("defaults.queue_id", MOCK_QUEUE_ID)
+
+    # Set the storage profile ID in the config; as someone may have by using `deadline config set`
+    config.set_setting("settings.storage_profile_id", PRE_STORAGE_PROFILE_ID)
+
+    # Use a temporary directory for the job bundle
+    with tempfile.TemporaryDirectory() as tmpdir, patch.object(boto3, "Session") as session_mock:
+        session_mock().client("deadline").create_job.return_value = MOCK_CREATE_JOB_RESPONSE
+        session_mock().client("deadline").get_job.return_value = MOCK_GET_JOB_RESPONSE
+        session_mock.reset_mock()
+
+        # Write a JSON template
+        with open(os.path.join(tmpdir, "template.json"), "w", encoding="utf8") as f:
+            f.write(MOCK_JOB_TEMPLATE_CASES["MINIMAL_JSON"][1])
+
+        runner = CliRunner()
+        with patch.object(api_module, "get_storage_profile_for_queue"):
+            result = runner.invoke(
+                main,
+                ["bundle", "submit", tmpdir, "--storage-profile-id", CLI_STORAGE_PROFILE_ID],
+            )
+
+    assert tmpdir in result.output
+    assert MOCK_CREATE_JOB_RESPONSE["jobId"] in result.output
+    assert MOCK_GET_JOB_RESPONSE["lifecycleStatusMessage"] in result.output
+    session_mock().client().create_job.assert_called_once_with(
+        farmId=MOCK_FARM_ID,
+        queueId=MOCK_QUEUE_ID,
+        template=ANY,
+        templateType="JSON",
+        priority=50,
+        storageProfileId=CLI_STORAGE_PROFILE_ID,
+    )
+    assert result.exit_code == 0
+    # Force a re-load from disk of the config object
+    with patch.object(config.config_file, "_should_read_config", return_value=True):
+        assert config.get_setting("settings.storage_profile_id") == PRE_STORAGE_PROFILE_ID
+
+
 @pytest.mark.parametrize("loading_method", [e.value for e in JobAttachmentsFileSystem] + [None])
 def test_cli_bundle_asset_load_method(fresh_deadline_config, temp_job_bundle_dir, loading_method):
     """

test/unit/deadline_job_attachments/test_download.py

@@ -2430,7 +2430,7 @@ def download_file(*args):
 
     with patch(
         f"{deadline.__package__}.job_attachments.download.download_file", side_effect=download_file
-    ):
+    ), patch(f"{deadline.__package__}.job_attachments.download.get_s3_client"):
         download_files_from_manifests(
             s3_bucket="s3_settings.s3BucketName",
             manifests_by_root={"/test": merged_manifest},