feat: add os_user and os_env_vars for use by deadline vfs (#104)

Signed-off-by: Nathan Matthews <[email protected]>

scripted_tests/set_file_permission_for_windows.py

@@ -94,6 +94,7 @@ def run_test():
         start_time = time.perf_counter()
 
         fs_permission_settings = WindowsFileSystemPermissionSettings(
+            os_user=args.target_user,
             os_group=args.group,
             dir_mode=dir_permission,
             file_mode=file_permission,

src/deadline/job_attachments/asset_sync.py

@@ -288,6 +288,7 @@ def sync_inputs(
         storage_profiles_path_mapping_rules: dict[str, str] = {},
         step_dependencies: Optional[list[str]] = None,
         on_downloading_files: Optional[Callable[[ProgressReportMetadata], bool]] = None,
+        os_env_vars: Dict[str, str] | None = None,
     ) -> Tuple[SummaryStatistics, List[Dict[str, str]]]:
         """
         Depending on the fileSystem in the Attachments this will perform two
@@ -312,6 +313,7 @@ def sync_inputs(
             on_downloading_files: a function that will be called with a ProgressReportMetadata object
                 for each file being downloaded. If the function returns False, the download will be
                 cancelled. If it returns True, the download will continue.
+            os_env_vars: environment variables to set for launched subprocesses
 
         Returns:
             COPIED / None : a tuple of (1) final summary statistics for file downloads,
@@ -400,6 +402,9 @@ def sync_inputs(
         if (
             attachments.fileSystem == JobAttachmentsFileSystem.VIRTUAL.value
             and sys.platform != "win32"
+            and fs_permission_settings is not None
+            and os_env_vars is not None
+            and "AWS_PROFILE" in os_env_vars
         ):
             try:
                 Fus3ProcessManager.find_fus3()
@@ -408,6 +413,8 @@ def sync_inputs(
                     manifests_by_root=merged_manifests_by_root,
                     boto3_session=self.session,
                     session_dir=session_dir,
+                    os_user=fs_permission_settings.os_user,  # type: ignore[union-attr]
+                    os_env_vars=os_env_vars,  # type: ignore[arg-type]
                     cas_prefix=s3_settings.full_cas_prefix(),
                 )
                 summary_statistics = SummaryStatistics()

src/deadline/job_attachments/download.py

@@ -778,6 +778,8 @@ def mount_vfs_from_manifests(
     manifests_by_root: dict[str, BaseAssetManifest],
     boto3_session: boto3.Session,
     session_dir: Path,
+    os_user: str,
+    os_env_vars: dict[str, str],
     cas_prefix: Optional[str] = None,
 ) -> None:
     """
@@ -787,7 +789,9 @@ def mount_vfs_from_manifests(
         s3_bucket: The name of the S3 bucket.
         manifests_by_root: a map from each local root path to a corresponding list of tuples of manifest contents and their path.
         boto3_session: The boto3 session to use.
-        session_dir: the directory that the session is going to use.z
+        session_dir: the directory that the session is going to use.
+        os_user: the user executing the job.
+        os_env_vars: environment variables to set for launched subprocesses
         cas_prefix: The CAS prefix of the files.
 
     Returns:
@@ -803,7 +807,13 @@ def mount_vfs_from_manifests(
             local_download_dir, [path.path for path in manifest.paths]  # type: ignore
         )
         vfs_manager: Fus3ProcessManager = Fus3ProcessManager(
-            s3_bucket, boto3_session.region_name, manifest_path, local_download_dir, cas_prefix
+            s3_bucket,
+            boto3_session.region_name,
+            manifest_path,
+            local_download_dir,
+            os_user,
+            os_env_vars,
+            cas_prefix,
         )
         vfs_manager.start(session_dir=session_dir)
 

src/deadline/job_attachments/fus3.py

@@ -9,7 +9,7 @@
 from pathlib import Path
 import threading
 from signal import SIGTERM
-from typing import List, Union, Optional
+from typing import Dict, List, Union, Optional
 
 from .exceptions import (
     Fus3ExecutableMissingError,
@@ -56,6 +56,8 @@ class Fus3ProcessManager(object):
     _asset_bucket: str
     _region: str
     _manifest_path: str
+    _os_user: str
+    _os_env_vars: Dict[str, str]
     _cas_prefix: Optional[str]
 
     def __init__(
@@ -64,6 +66,8 @@ def __init__(
         region: str,
         manifest_path: str,
         mount_point: str,
+        os_user: str,
+        os_env_vars: Dict[str, str],
         cas_prefix: Optional[str] = None,
     ):
         # TODO: Once Windows pathmapping is implemented we can remove this
@@ -78,6 +82,8 @@ def __init__(
         self._asset_bucket = asset_bucket
         self._region = region
         self._manifest_path = manifest_path
+        self._os_user = os_user
+        self._os_env_vars = os_env_vars
         self._cas_prefix = cas_prefix
 
     @classmethod
@@ -315,18 +321,19 @@ def get_cwd(cls) -> Union[os.PathLike, str]:
             )
         return Fus3ProcessManager.cwd_path
 
-    @classmethod
-    def get_launch_environ(cls) -> dict:
+    def get_launch_environ(self) -> dict:
         """
         Get the environment variables we'll pass to the launch command.
         :returns: dictionary of default environment variables with fus3 changes applied
         """
-        my_env = {**os.environ}
+        my_env = {**os.environ, **self._os_env_vars}
         my_env[
             "PATH"
         ] = f"{Fus3ProcessManager.find_fus3_link_dir()}{os.pathsep}{os.environ['PATH']}"
         my_env["LD_LIBRARY_PATH"] = Fus3ProcessManager.get_library_path()  # type: ignore[assignment]
 
+        my_env["AWS_CONFIG_FILE"] = str(Path(f"~{self._os_user}/.aws/config").expanduser())
+
         return my_env
 
     def start(self, session_dir: Path) -> None:
@@ -339,7 +346,7 @@ def start(self, session_dir: Path) -> None:
         log.info(f"Using mount_point {self._mount_point}")
         Fus3ProcessManager.create_mount_point(self._mount_point)
         start_command = self.build_launch_command(self._mount_point)
-        launch_env = Fus3ProcessManager.get_launch_environ()
+        launch_env = self.get_launch_environ()
         log.info(f"Launching fus3 with command {start_command}")
         log.info(f"Launching with environment {launch_env}")
 

src/deadline/job_attachments/models.py

@@ -291,11 +291,13 @@ class PosixFileSystemPermissionSettings:
     the directory or file's existing permissions.
 
     Attributes:
+        os_user (str): The target operating system user for ownership.
         os_group (str): The target operating system group for ownership.
         dir_mode (int): The permission mode to be added to directories.
         file_mode (int): The permission mode to be added to files.
     """
 
+    os_user: str
     os_group: str
     dir_mode: int
     file_mode: int

src/deadline/job_attachments/os_file_permission.py

@@ -20,11 +20,13 @@ class PosixFileSystemPermissionSettings:
     the directory or file's existing permissions.
 
     Attributes:
+        os_user (str): The target operating system user for ownership.
         os_group (str): The target operating system group for ownership.
         dir_mode (int): The permission mode to be added to directories.
         file_mode (int): The permission mode to be added to files.
     """
 
+    os_user: str
     os_group: str
     dir_mode: int
     file_mode: int
@@ -50,11 +52,13 @@ class WindowsFileSystemPermissionSettings:
     for Windows.
 
     Attributes:
+        os_user (str): The target operating system user or ownership.
         os_group (str): The target operating system group for ownership.
         dir_mode (WindowsPermissionEnum): The permission mode to be added to directories.
         file_mode (WindowsPermissionEnum): The permission mode to be added to files.
     """
 
+    os_user: str
     os_group: str
     dir_mode: WindowsPermissionEnum
     file_mode: WindowsPermissionEnum

test/unit/deadline_job_attachments/test_asset_sync.py

@@ -7,7 +7,7 @@
 import shutil
 from math import trunc
 from pathlib import Path
-from typing import Optional
+from typing import Optional, Dict
 from unittest.mock import ANY, MagicMock, call, mock_open, patch
 
 import boto3
@@ -17,6 +17,7 @@
 
 import deadline
 from deadline.job_attachments.asset_sync import AssetSync
+from deadline.job_attachments.os_file_permission import PosixFileSystemPermissionSettings
 from deadline.job_attachments.download import _progress_logger
 from deadline.job_attachments.exceptions import Fus3ExecutableMissingError
 from deadline.job_attachments.models import (
@@ -216,6 +217,15 @@ def test_sync_inputs_successful(
         session_dir = str(tmp_path)
         dest_dir = "assetroot-27bggh78dd2b568ab123"
         local_root = str(Path(session_dir) / dest_dir)
+        test_fs_permission_settings: PosixFileSystemPermissionSettings = (
+            PosixFileSystemPermissionSettings(
+                os_user="test-user",
+                os_group="test-group",
+                dir_mode=0o20,
+                file_mode=0o20,
+            )
+        )
+        os_env_vars: Dict[str, str] = {"AWS_PROFILE": "test-profile"}
         assert job.attachments
 
         # WHEN
@@ -245,6 +255,8 @@ def test_sync_inputs_successful(
                 job.jobId,
                 tmp_path,
                 on_downloading_files=mock_on_downloading_files,
+                fs_permission_settings=test_fs_permission_settings,
+                os_env_vars=os_env_vars,
             )
 
             # THEN

test/unit/deadline_job_attachments/test_download.py

@@ -743,6 +743,7 @@ def test_download_files_from_manifests_with_fs_permission_settings_posix(
         manifests_by_root = {str(tmp_path): manifest}
 
         fs_permission_settings = PosixFileSystemPermissionSettings(
+            os_user="test-user",
             os_group="test-group",
             dir_mode=0o20,
             file_mode=0o20,
@@ -809,6 +810,7 @@ def test_download_files_from_manifests_with_fs_permission_settings_windows(
         manifests_by_root = {str(tmp_path): manifest}
 
         fs_permission_settings = WindowsFileSystemPermissionSettings(
+            os_user="test-user",
             os_group="test-group",
             dir_mode=WindowsPermissionEnum.FULL_CONTROL,
             file_mode=WindowsPermissionEnum.FULL_CONTROL,
@@ -901,6 +903,7 @@ def test_download_files_from_manifests_have_correct_group_posix(
         manifests_by_root = {str(tmp_path): manifest}
 
         fs_permission_settings = PosixFileSystemPermissionSettings(
+            os_user="test-user",
             os_group=posix_target_group,
             dir_mode=0o20,
             file_mode=0o20,
@@ -980,6 +983,7 @@ def test_download_files_from_manifests_have_correct_group_windows(
 
         # Use a builtin group, so we can expect it to exist on any Windows machine
         fs_permission_settings = WindowsFileSystemPermissionSettings(
+            os_user="test-user",
             os_group="Users",
             dir_mode=WindowsPermissionEnum.FULL_CONTROL,
             file_mode=WindowsPermissionEnum.FULL_CONTROL,