fix: Adding os_user to FileSystemPermissionSettings for use by the

deadline vfs

Signed-off-by: Nathan Matthews <[email protected]>

scripted_tests/set_file_permission_for_windows.py

@@ -94,6 +94,7 @@ def run_test():
         start_time = time.perf_counter()
 
         fs_permission_settings = WindowsFileSystemPermissionSettings(
+            os_user=args.target_user,
             os_group=args.group,
             dir_mode=dir_permission,
             file_mode=file_permission,

src/deadline/job_attachments/asset_sync.py

@@ -400,6 +400,7 @@ def sync_inputs(
         if (
             attachments.fileSystem == JobAttachmentsFileSystem.VIRTUAL.value
             and sys.platform != "win32"
+            and fs_permission_settings is not None
         ):
             try:
                 Fus3ProcessManager.find_fus3()
@@ -408,6 +409,8 @@ def sync_inputs(
                     manifests_by_root=merged_manifests_by_root,
                     boto3_session=self.session,
                     session_dir=session_dir,
+                    queue_id=queue_id,
+                    os_user=fs_permission_settings.os_user,
                     cas_prefix=s3_settings.full_cas_prefix(),
                 )
                 summary_statistics = SummaryStatistics()

src/deadline/job_attachments/download.py

@@ -778,6 +778,8 @@ def mount_vfs_from_manifests(
     manifests_by_root: dict[str, BaseAssetManifest],
     boto3_session: boto3.Session,
     session_dir: Path,
+    queue_id: str,
+    os_user: str,
     cas_prefix: Optional[str] = None,
 ) -> None:
     """
@@ -788,6 +790,8 @@ def mount_vfs_from_manifests(
         manifests_by_root: a map from each local root path to a corresponding list of tuples of manifest contents and their path.
         boto3_session: The boto3 session to use.
         session_dir: the directory that the session is going to use.z
+        queue_id: the ID of the queue.
+        os_user: the user executing the job.
         cas_prefix: The CAS prefix of the files.
 
     Returns:
@@ -803,7 +807,13 @@ def mount_vfs_from_manifests(
             local_download_dir, [path.path for path in manifest.paths]  # type: ignore
         )
         vfs_manager: Fus3ProcessManager = Fus3ProcessManager(
-            s3_bucket, boto3_session.region_name, manifest_path, local_download_dir, cas_prefix
+            s3_bucket,
+            boto3_session.region_name,
+            manifest_path,
+            local_download_dir,
+            queue_id,
+            os_user,
+            cas_prefix,
         )
         vfs_manager.start(session_dir=session_dir)
 

src/deadline/job_attachments/fus3.py

@@ -56,6 +56,8 @@ class Fus3ProcessManager(object):
     _asset_bucket: str
     _region: str
     _manifest_path: str
+    _queue_id: str
+    _os_user: str
     _cas_prefix: Optional[str]
 
     def __init__(
@@ -64,6 +66,8 @@ def __init__(
         region: str,
         manifest_path: str,
         mount_point: str,
+        queue_id: str,
+        os_user: str,
         cas_prefix: Optional[str] = None,
     ):
         # TODO: Once Windows pathmapping is implemented we can remove this
@@ -78,6 +82,8 @@ def __init__(
         self._asset_bucket = asset_bucket
         self._region = region
         self._manifest_path = manifest_path
+        self._queue_id = queue_id
+        self._os_user = os_user
         self._cas_prefix = cas_prefix
 
     @classmethod
@@ -315,8 +321,7 @@ def get_cwd(cls) -> Union[os.PathLike, str]:
             )
         return Fus3ProcessManager.cwd_path
 
-    @classmethod
-    def get_launch_environ(cls) -> dict:
+    def get_launch_environ(self) -> dict:
         """
         Get the environment variables we'll pass to the launch command.
         :returns: dictionary of default environment variables with fus3 changes applied
@@ -327,6 +332,9 @@ def get_launch_environ(cls) -> dict:
         ] = f"{Fus3ProcessManager.find_fus3_link_dir()}{os.pathsep}{os.environ['PATH']}"
         my_env["LD_LIBRARY_PATH"] = Fus3ProcessManager.get_library_path()  # type: ignore[assignment]
 
+        my_env["AWS_CONFIG_FILE"] = Path(f"~{self._os_user}/.aws/config").expanduser().as_posix()
+        my_env["AWS_PROFILE"] = f"deadline-{self._queue_id}"
+
         return my_env
 
     def start(self, session_dir: Path) -> None:
@@ -339,7 +347,7 @@ def start(self, session_dir: Path) -> None:
         log.info(f"Using mount_point {self._mount_point}")
         Fus3ProcessManager.create_mount_point(self._mount_point)
         start_command = self.build_launch_command(self._mount_point)
-        launch_env = Fus3ProcessManager.get_launch_environ()
+        launch_env = self.get_launch_environ()
         log.info(f"Launching fus3 with command {start_command}")
         log.info(f"Launching with environment {launch_env}")
 

src/deadline/job_attachments/models.py

@@ -291,11 +291,13 @@ class PosixFileSystemPermissionSettings:
     the directory or file's existing permissions.
 
     Attributes:
+        os_user (str): The target operating system user for ownership.
         os_group (str): The target operating system group for ownership.
         dir_mode (int): The permission mode to be added to directories.
         file_mode (int): The permission mode to be added to files.
     """
 
+    os_user: str
     os_group: str
     dir_mode: int
     file_mode: int

src/deadline/job_attachments/os_file_permission.py

@@ -20,11 +20,13 @@ class PosixFileSystemPermissionSettings:
     the directory or file's existing permissions.
 
     Attributes:
+        os_user (str): The target operating system user for ownership.
         os_group (str): The target operating system group for ownership.
         dir_mode (int): The permission mode to be added to directories.
         file_mode (int): The permission mode to be added to files.
     """
 
+    os_user: str
     os_group: str
     dir_mode: int
     file_mode: int
@@ -50,11 +52,13 @@ class WindowsFileSystemPermissionSettings:
     for Windows.
 
     Attributes:
+        os_user (str): The target operating system user or ownership.
         os_group (str): The target operating system group for ownership.
         dir_mode (WindowsPermissionEnum): The permission mode to be added to directories.
         file_mode (WindowsPermissionEnum): The permission mode to be added to files.
     """
 
+    os_user: str
     os_group: str
     dir_mode: WindowsPermissionEnum
     file_mode: WindowsPermissionEnum

test/unit/deadline_job_attachments/test_asset_sync.py

@@ -17,6 +17,7 @@
 
 import deadline
 from deadline.job_attachments.asset_sync import AssetSync
+from deadline.job_attachments.os_file_permission import PosixFileSystemPermissionSettings
 from deadline.job_attachments.download import _progress_logger
 from deadline.job_attachments.exceptions import Fus3ExecutableMissingError
 from deadline.job_attachments.models import (
@@ -216,6 +217,14 @@ def test_sync_inputs_successful(
         session_dir = str(tmp_path)
         dest_dir = "assetroot-27bggh78dd2b568ab123"
         local_root = str(Path(session_dir) / dest_dir)
+        test_fs_permission_settings: PosixFileSystemPermissionSettings = (
+            PosixFileSystemPermissionSettings(
+                os_user="test-user",
+                os_group="test-group",
+                dir_mode=0o20,
+                file_mode=0o20,
+            )
+        )
         assert job.attachments
 
         # WHEN
@@ -245,6 +254,7 @@ def test_sync_inputs_successful(
                 job.jobId,
                 tmp_path,
                 on_downloading_files=mock_on_downloading_files,
+                fs_permission_settings=test_fs_permission_settings,
             )
 
             # THEN

test/unit/deadline_job_attachments/test_download.py

@@ -743,6 +743,7 @@ def test_download_files_from_manifests_with_fs_permission_settings_posix(
         manifests_by_root = {str(tmp_path): manifest}
 
         fs_permission_settings = PosixFileSystemPermissionSettings(
+            os_user="test-user",
             os_group="test-group",
             dir_mode=0o20,
             file_mode=0o20,
@@ -809,6 +810,7 @@ def test_download_files_from_manifests_with_fs_permission_settings_windows(
         manifests_by_root = {str(tmp_path): manifest}
 
         fs_permission_settings = WindowsFileSystemPermissionSettings(
+            os_user="test-user",
             os_group="test-group",
             dir_mode=WindowsPermissionEnum.FULL_CONTROL,
             file_mode=WindowsPermissionEnum.FULL_CONTROL,
@@ -901,6 +903,7 @@ def test_download_files_from_manifests_have_correct_group_posix(
         manifests_by_root = {str(tmp_path): manifest}
 
         fs_permission_settings = PosixFileSystemPermissionSettings(
+            os_user="test-user",
             os_group=posix_target_group,
             dir_mode=0o20,
             file_mode=0o20,
@@ -980,6 +983,7 @@ def test_download_files_from_manifests_have_correct_group_windows(
 
         # Use a builtin group, so we can expect it to exist on any Windows machine
         fs_permission_settings = WindowsFileSystemPermissionSettings(
+            os_user="test-user",
             os_group="Users",
             dir_mode=WindowsPermissionEnum.FULL_CONTROL,
             file_mode=WindowsPermissionEnum.FULL_CONTROL,

test/unit/deadline_job_attachments/test_fus3.py

@@ -76,6 +76,8 @@ def test_init_fails_on_windows(self) -> None:
                 region=os.environ["AWS_DEFAULT_REGION"],
                 manifest_path="/test/manifest/path",
                 mount_point="/test/mount/point",
+                queue_id="test-queue",
+                os_user="test-user",
             )
 
     def test_build_launch_command(
@@ -94,6 +96,8 @@ def test_build_launch_command(
             region=os.environ["AWS_DEFAULT_REGION"],
             manifest_path=manifest_path,
             mount_point=local_root,
+            queue_id="test-queue",
+            os_user="test-user",
         )
 
         test_executable = os.environ[FUS3_PATH_ENV_VAR] + DEADLINE_VFS_EXECUTABLE_SCRIPT
@@ -124,6 +128,8 @@ def test_build_launch_command(
             region=os.environ["AWS_DEFAULT_REGION"],
             manifest_path=manifest_path,
             mount_point=local_root,
+            queue_id="test-queue",
+            os_user="test-user",
             cas_prefix=test_CAS_prefix,
         )
 
@@ -166,6 +172,8 @@ def test_build_launch_command_fallback(
             region=os.environ["AWS_DEFAULT_REGION"],
             manifest_path=manifest_path,
             mount_point=local_root,
+            queue_id="test-queue",
+            os_user="test-user",
         )
 
         test_executable = os.environ[FUS3_PATH_ENV_VAR] + FUS3_EXECUTABLE_SCRIPT
@@ -196,6 +204,8 @@ def test_build_launch_command_fallback(
             region=os.environ["AWS_DEFAULT_REGION"],
             manifest_path=manifest_path,
             mount_point=local_root,
+            queue_id="test-queue",
+            os_user="test-user",
             cas_prefix=test_CAS_prefix,
         )
 
@@ -238,6 +248,8 @@ def test_find_fus3_with_env_set(
             region=os.environ["AWS_DEFAULT_REGION"],
             manifest_path=manifest_path,
             mount_point=local_root,
+            queue_id="test-queue",
+            os_user="test-user",
         )
 
         # verify which is only called when class path is not set
@@ -289,6 +301,8 @@ def test_find_fus3_with_deadline_env_set(
             region=os.environ["AWS_DEFAULT_REGION"],
             manifest_path=manifest_path,
             mount_point=local_root,
+            queue_id="test-queue",
+            os_user="test-user",
         )
 
         # verify which is only called when class path is not set
@@ -339,6 +353,8 @@ def test_find_fus3_fallback(
             region=os.environ["AWS_DEFAULT_REGION"],
             manifest_path=manifest_path,
             mount_point=local_root,
+            queue_id="test-queue",
+            os_user="test-user",
         )
 
         # Verify that fus3 can be picked up if deadline_vfs is not found
@@ -366,6 +382,8 @@ def find_fus3_with_env_not_set(
             region=os.environ["AWS_DEFAULT_REGION"],
             manifest_path=manifest_path,
             mount_point=local_root,
+            queue_id="test-queue",
+            os_user="test-user",
         )
 
         bin_check = os.path.join(os.getcwd(), f"bin/{DEADLINE_VFS_EXECUTABLE}")
@@ -401,6 +419,8 @@ def test_find_library_path(
             region=os.environ["AWS_DEFAULT_REGION"],
             manifest_path=manifest_path,
             mount_point=local_root,
+            queue_id="test-queue",
+            os_user="test-user",
         )
 
         with patch(
@@ -432,6 +452,8 @@ def test_find_fus3_launch_script_with_env_set(
             region=os.environ["AWS_DEFAULT_REGION"],
             manifest_path=manifest_path,
             mount_point=local_root,
+            queue_id="test-queue",
+            os_user="test-user",
         )
 
         with patch(
@@ -473,6 +495,8 @@ def test_find_fus3_launch_script_fallback(
             region=os.environ["AWS_DEFAULT_REGION"],
             manifest_path=manifest_path,
             mount_point=local_root,
+            queue_id="test-queue",
+            os_user="test-user",
         )
 
         with patch(
@@ -503,6 +527,8 @@ def test_find_fus3_launch_script_with_env_not_set(
             region=os.environ["AWS_DEFAULT_REGION"],
             manifest_path=manifest_path,
             mount_point=local_root,
+            queue_id="test-queue",
+            os_user="test-user",
         )
 
         with patch(
@@ -544,6 +570,8 @@ def test_create_mount_point(
             region=os.environ["AWS_DEFAULT_REGION"],
             manifest_path=manifest_path,
             mount_point=local_root,
+            queue_id="test-queue",
+            os_user="test-user",
         )
 
         # Verify mount point is created and others have rwx access to it
@@ -574,12 +602,16 @@ def test_pids_recorded_and_killed(
             region=os.environ["AWS_DEFAULT_REGION"],
             manifest_path=manifest_path1,
             mount_point=local_root1,
+            queue_id="test-queue",
+            os_user="test-user",
         )
         process_manager2: Fus3ProcessManager = Fus3ProcessManager(
             asset_bucket=self.s3_settings.s3BucketName,
             region=os.environ["AWS_DEFAULT_REGION"],
             manifest_path=manifest_path2,
             mount_point=local_root2,
+            queue_id="test-queue",
+            os_user="test-user",
         )
 
         with patch(
@@ -597,7 +629,10 @@ def test_pids_recorded_and_killed(
             return_value=True,
         ), patch(
             f"{deadline.__package__}.job_attachments.fus3.subprocess.run"
-        ) as mock_subprocess_run:
+        ) as mock_subprocess_run, patch(
+            f"{deadline.__package__}.job_attachments.fus3.Fus3ProcessManager.get_launch_environ",
+            return_value=os.environ,
+        ):
             # start first mock fus3 process
             mock_subprocess = MagicMock()
             mock_subprocess.pid = test_pid1
@@ -654,6 +689,8 @@ def test_process_output_captured(
             region=os.environ["AWS_DEFAULT_REGION"],
             manifest_path=manifest_path1,
             mount_point=local_root1,
+            queue_id="test-queue",
+            os_user="test-user",
         )
 
         with patch(
@@ -669,7 +706,10 @@ def test_process_output_captured(
             return_value=True,
         ), patch(
             f"{deadline.__package__}.job_attachments.fus3.log"
-        ) as mock_logger:
+        ) as mock_logger, patch(
+            f"{deadline.__package__}.job_attachments.fus3.Fus3ProcessManager.get_launch_environ",
+            return_value=os.environ,
+        ):
             call_count = 0
             exception_count = 0
             signal = threading.Semaphore(0)