From 6b60cf2f1e1b57b6ef6160375e88ebcfcad5de2d Mon Sep 17 00:00:00 2001 From: Josh Usiskin <56369778+jusiskin@users.noreply.github.com> Date: Fri, 6 Dec 2024 15:32:09 +0000 Subject: [PATCH] fix: TOML-aware configuration changes Signed-off-by: Josh Usiskin <56369778+jusiskin@users.noreply.github.com> --- .github/workflows/code_quality.yml | 23 +- .github/workflows/mainline_e2e_canary.yml | 11 +- .github/workflows/mainline_e2e_test.yml | 11 +- .github/workflows/release_bump.yml | 13 +- .github/workflows/release_e2e_canary.yml | 11 +- DEVELOPMENT.md | 13 + hatch.toml | 2 +- pyproject.toml | 1 + src/deadline_worker_agent/config/__main__.py | 227 +++++++++++ .../config/config_file.py | 277 ++++++++++++- .../config/toml_comment.py | 81 ++++ .../installer/__init__.py | 2 + .../installer/install.sh | 79 ++-- .../installer/win_installer.py | 143 +++---- .../windows/win_credentials_resolver.py | 8 +- .../windows/win_logon.py | 5 +- .../windows/win_service.py | 27 -- .../windows/win_session.py | 30 ++ src/deadline_worker_agent/worker.py | 3 +- test/integ/{installer => config}/__init__.py | 0 test/integ/config/conftest.py | 55 +++ .../allow_ec2_instance_profile/input.toml | 26 ++ .../allow_ec2_instance_profile/output.toml | 26 ++ .../config/data/commented/farm_id/input.toml | 12 + .../config/data/commented/farm_id/output.toml | 12 + .../config/data/commented/fleet_id/input.toml | 12 + .../data/commented/fleet_id/output.toml | 12 + .../commented/shutdown_on_stop/input.toml | 15 + .../commented/shutdown_on_stop/output.toml | 15 + .../commented/windows_job_user/input.toml | 13 + .../commented/windows_job_user/output.toml | 13 + .../allow_ec2_instance_profile/input.toml | 26 ++ .../allow_ec2_instance_profile/output.toml | 26 ++ .../config/data/existing/farm_id/input.toml | 12 + .../config/data/existing/farm_id/output.toml | 12 + .../config/data/existing/fleet_id/input.toml | 12 + .../config/data/existing/fleet_id/output.toml | 12 + .../data/existing/shutdown_on_stop/input.toml | 15 + .../existing/shutdown_on_stop/output.toml | 15 + .../data/existing/windows_job_user/input.toml | 13 + .../existing/windows_job_user/output.toml | 13 + .../allow_ec2_instance_profile/input.toml | 7 + .../allow_ec2_instance_profile/output.toml | 32 ++ .../config/data/missing/farm_id/input.toml | 7 + .../config/data/missing/farm_id/output.toml | 18 + .../config/data/missing/fleet_id/input.toml | 7 + .../config/data/missing/fleet_id/output.toml | 18 + .../data/missing/shutdown_on_stop/input.toml | 7 + .../data/missing/shutdown_on_stop/output.toml | 21 + .../data/missing/windows_job_user/input.toml | 7 + .../data/missing/windows_job_user/output.toml | 19 + .../allow_ec2_instance_profile/input.toml | 26 ++ .../allow_ec2_instance_profile/output.toml | 26 ++ .../config/data/unset/farm_id/input.toml | 12 + .../config/data/unset/farm_id/output.toml | 12 + .../config/data/unset/fleet_id/input.toml | 12 + .../config/data/unset/fleet_id/output.toml | 12 + .../data/unset/shutdown_on_stop/input.toml | 15 + .../data/unset/shutdown_on_stop/output.toml | 15 + .../data/unset/windows_job_user/input.toml | 13 + .../data/unset/windows_job_user/output.toml | 13 + test/integ/config/test_config_cli.py | 369 ++++++++++++++++++ test/integ/config/test_config_file.py | 323 +++++++++++++++ test/integ/conftest.py | 11 + test/integ/windows/__init__.py | 1 + .../test_installer.py} | 3 - test/unit/install/test_install.py | 3 + test/unit/windows/test_win_logon.py | 4 +- test/unit/windows/test_win_service.py | 65 --- test/unit/windows/test_win_session.py | 73 ++++ 70 files changed, 2212 insertions(+), 273 deletions(-) create mode 100644 src/deadline_worker_agent/config/__main__.py create mode 100644 src/deadline_worker_agent/config/toml_comment.py create mode 100644 src/deadline_worker_agent/windows/win_session.py rename test/integ/{installer => config}/__init__.py (100%) create mode 100644 test/integ/config/conftest.py create mode 100644 test/integ/config/data/commented/allow_ec2_instance_profile/input.toml create mode 100644 test/integ/config/data/commented/allow_ec2_instance_profile/output.toml create mode 100644 test/integ/config/data/commented/farm_id/input.toml create mode 100644 test/integ/config/data/commented/farm_id/output.toml create mode 100644 test/integ/config/data/commented/fleet_id/input.toml create mode 100644 test/integ/config/data/commented/fleet_id/output.toml create mode 100644 test/integ/config/data/commented/shutdown_on_stop/input.toml create mode 100644 test/integ/config/data/commented/shutdown_on_stop/output.toml create mode 100644 test/integ/config/data/commented/windows_job_user/input.toml create mode 100644 test/integ/config/data/commented/windows_job_user/output.toml create mode 100644 test/integ/config/data/existing/allow_ec2_instance_profile/input.toml create mode 100644 test/integ/config/data/existing/allow_ec2_instance_profile/output.toml create mode 100644 test/integ/config/data/existing/farm_id/input.toml create mode 100644 test/integ/config/data/existing/farm_id/output.toml create mode 100644 test/integ/config/data/existing/fleet_id/input.toml create mode 100644 test/integ/config/data/existing/fleet_id/output.toml create mode 100644 test/integ/config/data/existing/shutdown_on_stop/input.toml create mode 100644 test/integ/config/data/existing/shutdown_on_stop/output.toml create mode 100644 test/integ/config/data/existing/windows_job_user/input.toml create mode 100644 test/integ/config/data/existing/windows_job_user/output.toml create mode 100644 test/integ/config/data/missing/allow_ec2_instance_profile/input.toml create mode 100644 test/integ/config/data/missing/allow_ec2_instance_profile/output.toml create mode 100644 test/integ/config/data/missing/farm_id/input.toml create mode 100644 test/integ/config/data/missing/farm_id/output.toml create mode 100644 test/integ/config/data/missing/fleet_id/input.toml create mode 100644 test/integ/config/data/missing/fleet_id/output.toml create mode 100644 test/integ/config/data/missing/shutdown_on_stop/input.toml create mode 100644 test/integ/config/data/missing/shutdown_on_stop/output.toml create mode 100644 test/integ/config/data/missing/windows_job_user/input.toml create mode 100644 test/integ/config/data/missing/windows_job_user/output.toml create mode 100644 test/integ/config/data/unset/allow_ec2_instance_profile/input.toml create mode 100644 test/integ/config/data/unset/allow_ec2_instance_profile/output.toml create mode 100644 test/integ/config/data/unset/farm_id/input.toml create mode 100644 test/integ/config/data/unset/farm_id/output.toml create mode 100644 test/integ/config/data/unset/fleet_id/input.toml create mode 100644 test/integ/config/data/unset/fleet_id/output.toml create mode 100644 test/integ/config/data/unset/shutdown_on_stop/input.toml create mode 100644 test/integ/config/data/unset/shutdown_on_stop/output.toml create mode 100644 test/integ/config/data/unset/windows_job_user/input.toml create mode 100644 test/integ/config/data/unset/windows_job_user/output.toml create mode 100644 test/integ/config/test_config_cli.py create mode 100644 test/integ/config/test_config_file.py create mode 100644 test/integ/conftest.py create mode 100644 test/integ/windows/__init__.py rename test/integ/{installer/test_windows_installer.py => windows/test_installer.py} (99%) create mode 100644 test/unit/windows/test_win_session.py diff --git a/.github/workflows/code_quality.yml b/.github/workflows/code_quality.yml index 69a2c69f..41e9f1d9 100644 --- a/.github/workflows/code_quality.yml +++ b/.github/workflows/code_quality.yml @@ -10,7 +10,7 @@ on: type: string jobs: - Test: + UnitTest: name: Python strategy: matrix: @@ -19,4 +19,23 @@ jobs: uses: aws-deadline/.github/.github/workflows/reusable_python_build.yml@mainline with: os: ${{ matrix.os }} - python-version: ${{ matrix.python-version }} \ No newline at end of file + python-version: ${{ matrix.python-version }} + IntegrationTests: + name: Integration Tests - {{ matrix.os }}, python {{ matrix.python-version }} + needs: UnitTest + runs-on: {{ matrix.os }} + permissions: + contents: read + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + python-version: ['3.9', '3.10', '3.11'] + steps: + - uses: actions/checkout@v4 + + - name: Install Hatch + run: | + pip install --upgrade hatch + + - name: Run Integration Tests + run: hatch run integ-test \ No newline at end of file diff --git a/.github/workflows/mainline_e2e_canary.yml b/.github/workflows/mainline_e2e_canary.yml index e8412608..2139f5d9 100644 --- a/.github/workflows/mainline_e2e_canary.yml +++ b/.github/workflows/mainline_e2e_canary.yml @@ -6,13 +6,14 @@ on: workflow_dispatch: jobs: - WindowsIntegrationTests: - name: Windows Integration Tests - runs-on: windows-latest + IntegrationTests: + name: Integration Tests - {{ matrix.os }}, python {{ matrix.python-version }} + runs-on: {{ matrix.os }} permissions: contents: read strategy: matrix: + os: [ubuntu-latest, windows-latest] python-version: ['3.9', '3.10', '3.11'] steps: - uses: actions/checkout@v4 @@ -21,8 +22,8 @@ jobs: run: | pip install --upgrade hatch - - name: Run Windows Integration Tests - run: hatch run windows-integ-test + - name: Run Integration Tests + run: hatch run integ-test MainlineLinuxE2ECanary: name: Mainline Linux Canary diff --git a/.github/workflows/mainline_e2e_test.yml b/.github/workflows/mainline_e2e_test.yml index f41b6069..8bb0e55c 100644 --- a/.github/workflows/mainline_e2e_test.yml +++ b/.github/workflows/mainline_e2e_test.yml @@ -9,13 +9,14 @@ on: jobs: - WindowsIntegrationTests: - name: Windows Integration Tests - runs-on: windows-latest + IntegrationTests: + name: Integration Tests - {{ matrix.os }}, python {{ matrix.python-version }} + runs-on: {{ matrix.os }} permissions: contents: read strategy: matrix: + os: [ubuntu-latest, windows-latest] python-version: ['3.9', '3.10', '3.11'] steps: - uses: actions/checkout@v4 @@ -24,8 +25,8 @@ jobs: run: | pip install --upgrade hatch - - name: Run Windows Integration Tests - run: hatch run windows-integ-test + - name: Run Integration Tests + run: hatch run integ-test MainlineLinuxE2ETest: name: Linux E2E Test diff --git a/.github/workflows/release_bump.yml b/.github/workflows/release_bump.yml index 5d7ba94d..23b90aa9 100644 --- a/.github/workflows/release_bump.yml +++ b/.github/workflows/release_bump.yml @@ -20,15 +20,16 @@ jobs: with: branch: mainline - WindowsIntegrationTests: + IntegrationTests: needs: UnitTests - name: Windows Integration Tests - runs-on: windows-latest + name: Integration Tests - {{ matrix.os }}, python {{ matrix.python-version }} + runs-on: {{ matrix.os }} permissions: id-token: write contents: read strategy: matrix: + os: [ubuntu-latest, windows-latest] python-version: ['3.9', '3.10', '3.11'] env: PYTHON: ${{ matrix.python-version }} @@ -39,8 +40,8 @@ jobs: run: | pip install --upgrade hatch - - name: Run Windows Integration Tests - run: hatch run windows-integ-test + - name: Run Integration Tests + run: hatch run integ-test LinuxE2ETests: needs: UnitTests @@ -75,7 +76,7 @@ jobs: group: windowse2e Bump: - needs: [LinuxE2ETests, WindowsE2ETests, WindowsIntegrationTests] + needs: [LinuxE2ETests, WindowsE2ETests, IntegrationTests] name: Version Bump uses: aws-deadline/.github/.github/workflows/reusable_bump.yml@mainline secrets: inherit diff --git a/.github/workflows/release_e2e_canary.yml b/.github/workflows/release_e2e_canary.yml index f6432a9a..681aa542 100644 --- a/.github/workflows/release_e2e_canary.yml +++ b/.github/workflows/release_e2e_canary.yml @@ -6,13 +6,14 @@ on: workflow_dispatch: jobs: - WindowsIntegrationTests: - name: Windows Integration Tests - runs-on: windows-latest + IntegrationTests: + name: Integration Tests - {{ matrix.os }}, python {{ matrix.python-version }} + runs-on: {{ matrix.os }} permissions: contents: read strategy: matrix: + os: [ubuntu-latest, windows-latest] python-version: ['3.9', '3.10', '3.11'] steps: - uses: actions/checkout@v4 @@ -21,8 +22,8 @@ jobs: run: | pip install --upgrade hatch - - name: Run Windows Integration Tests - run: hatch run windows-integ-test + - name: Run Integration Tests + run: hatch run integ-test ReleaseLinuxE2ECanary: name: Release Linux Canary diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 80f3a7eb..4f1619bd 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -132,6 +132,19 @@ To stop the agent, simply run: docker exec test_worker_agent /home/agentuser/term_agent.sh ``` +### Running Worker Agent Integration Tests + +The worker agent has integration tests that run locally on the host machine they are run from. +These tests cover integration with the host operating system and file-system. If you are making +changes that apply to both Windows and Linux, you will need to test your changes on both a Linux +host and a Windows host. + +To run the tests, run: + +```sh +hatch run integ-test +``` + ### Running Worker Agent E2E Tests The worker agent has end-to-end tests that run the agent on ec2 instances with the live Deadline Cloud service. These tests diff --git a/hatch.toml b/hatch.toml index 60ccaa1f..f02ba298 100644 --- a/hatch.toml +++ b/hatch.toml @@ -12,7 +12,7 @@ test = "pytest test/unit --numprocesses=auto {args}" version = "hatch version" metadata = "hatch project metadata {args:}" e2e-test= "pytest --no-cov test/e2e {args:}" -windows-integ-test = "pytest --no-cov test/integ/installer {args:}" +integ-test = "pytest --no-cov test/integ {args:}" typing = "mypy {args:src test}" style = [ "ruff check {args:.}", diff --git a/pyproject.toml b/pyproject.toml index 78144c53..e4ade74b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ dependencies = [ "openjd-sessions >= 0.9.0,< 0.10", # tomli became tomllib in standard library in Python 3.11 "tomli == 2.0.* ; python_version<'3.11'", + "tomlkit == 0.13.*", "typing_extensions ~= 4.8", "psutil >= 5.9,< 7.0", "pydantic ~= 1.10.0", diff --git a/src/deadline_worker_agent/config/__main__.py b/src/deadline_worker_agent/config/__main__.py new file mode 100644 index 00000000..826821a0 --- /dev/null +++ b/src/deadline_worker_agent/config/__main__.py @@ -0,0 +1,227 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +from __future__ import annotations +import argparse +from logging import basicConfig, getLogger, INFO +from pathlib import Path + +from .config_file import ( + ConfigFile, + SettingModification, + ModifiableSetting, +) + + +logger = getLogger(__name__) + + +class ParsedArguments(argparse.Namespace): + backup: bool + """Whether to backup the existing worker agent configuration file. The backup is created + with along-side the worker agent configuration file with a .bak extension added""" + + config_path: Path | None = None + """The path to the worker agent configuration file to be modified""" + + farm_id: str | None = None + """The unique identifier for the Deadline Cloud farm that the worker belongs to""" + + fleet_id: str | None = None + """The unique identifier for the Deadline Cloud fleet that the worker belongs to""" + + shutdown_on_stop: bool | None = None + """Whether the worker agent will attempt to shutdown the worker host when the service instructs + the worker to stop""" + + allow_ec2_instance_profile: bool | None = None + """Whether or not the worker agent will allow being started if an EC2 instance profile is + detected""" + + windows_job_user: str | bool | None = None + """A Windows username to override the queue jobRunAs configuration. If False, then no + modification is made. If None, then the setting is unset.""" + + +def create_argument_parser() -> argparse.ArgumentParser: + """Creates the argparse ArgumentParser for the deadline_worker_agent.config module""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--config-path", + type=lambda x: Path(x) if x is not None else None, + help=argparse.SUPPRESS, + ) + worker_group = parser.add_argument_group("Worker", "Settings about the Deadline Cloud Worker") + + worker_group.add_argument( + "--farm-id", + help="The unique identifier for the Deadline Cloud farm", + required=False, + ) + worker_group.add_argument( + "--fleet-id", + help="The unique identifier for the Deadline Cloud fleet", + required=False, + ) + + aws_group = parser.add_argument_group("AWS", "Settings related to AWS and EC2 hosts") + parser.set_defaults(allow_ec2_instance_profile=None) + allow_instance_profile_group = aws_group.add_mutually_exclusive_group() + allow_instance_profile_group.add_argument( + "--allow-ec2-instance-profile", + help=( + "The worker agent will be allowed to run with an EC2 instance profile associated " + "to the host instance." + ), + action="store_const", + const=True, + required=False, + ) + allow_instance_profile_group.add_argument( + "--no-allow-ec2-instance-profile", + help=( + "The worker agent will fail to run with an EC2 instance profile associated to the " + "host instance." + ), + action="store_const", + dest="allow_ec2_instance_profile", + const=False, + required=False, + ) + + os_group = parser.add_argument_group( + "Operating System", "Settings related to the host operating system" + ) + + parser.set_defaults(shutdown_on_stop=None) + shutdown_on_stop_group = os_group.add_mutually_exclusive_group() + shutdown_on_stop_group.add_argument( + "--shutdown-on-stop", + help=( + "The worker agent will attempt to shutdown the host machine when the service tells " + "the worker to stop" + ), + action="store_const", + const=True, + required=False, + ) + shutdown_on_stop_group.add_argument( + "--no-shutdown-on-stop", + help=( + "The worker agent will not attempt to shutdown the host machine when the service " + "tells the worker to stop" + ), + action="store_const", + dest="shutdown_on_stop", + required=False, + const=False, + ) + + parser.set_defaults(windows_job_user=False) + windows_job_user_group = parser.add_mutually_exclusive_group() + windows_job_user_group.add_argument( + "--windows-job-user", + help=( + "On Windows, this refers to the username of a local account that will be used " + "to run jobs. This overrides the queue jobRunAsUser configuration." + ), + required=False, + ) + windows_job_user_group.add_argument( + "--no-windows-job-user", + help=( + "On Windows, this turns off a previous job user override. Jobs will then run " + "using the queue jobRunAsUser configuration." + ), + required=False, + dest="windows_job_user", + action="store_const", + const=None, + ) + + parser.add_argument( + "--no-backup", + help=( + "Do not create a backup. The default behavior is to create a backup of the " + 'configuration file to a path an added ".bak" extension.' + ), + action="store_false", + dest="backup", + ) + return parser + + +def args_to_setting_modifications(parsed_args: ParsedArguments) -> list[SettingModification]: + settings_to_modify = list[SettingModification]() + + if parsed_args.farm_id is not None: + settings_to_modify.append( + SettingModification( + setting=ModifiableSetting.FARM_ID, + value=parsed_args.farm_id, + ) + ) + if parsed_args.fleet_id is not None: + settings_to_modify.append( + SettingModification( + setting=ModifiableSetting.FLEET_ID, + value=parsed_args.fleet_id, + ) + ) + if parsed_args.allow_ec2_instance_profile is not None: + settings_to_modify.append( + SettingModification( + setting=ModifiableSetting.ALLOW_EC2_INSTANCE_PROFILE, + value=parsed_args.allow_ec2_instance_profile, + ) + ) + if parsed_args.shutdown_on_stop is not None: + settings_to_modify.append( + SettingModification( + setting=ModifiableSetting.SHUTDOWN_ON_STOP, + value=parsed_args.shutdown_on_stop, + ) + ) + if parsed_args.windows_job_user is not False: + if isinstance(parsed_args.windows_job_user, str) or parsed_args.windows_job_user is None: + settings_to_modify.append( + SettingModification( + setting=ModifiableSetting.WINDOWS_JOB_USER, + value=parsed_args.windows_job_user, + ) + ) + else: + raise NotImplementedError( + f"Unexpected value for windows_job_user: {parsed_args.windows_job_user}" + ) + + return settings_to_modify + + +def main() -> None: + basicConfig(format="%(msg)s", level=INFO) + parser = create_argument_parser() + args = parser.parse_args(namespace=ParsedArguments()) + + if (config_path := args.config_path) is None: + config_path = ConfigFile.get_config_path() + + settings_to_modify = args_to_setting_modifications(args) + + if not settings_to_modify: + parser.error("No settings to modify") + else: + logger.info("The following settings will be modified:") + for setting_to_modify in settings_to_modify: + logger.info( + f" {setting_to_modify.setting.value.setting_name} = {setting_to_modify.value}" + ) + + ConfigFile.modify_config_file_settings( + settings_to_modify=settings_to_modify, + backup=args.backup, + config_path=config_path, + ) + + +if __name__ == "__main__": + main() diff --git a/src/deadline_worker_agent/config/config_file.py b/src/deadline_worker_agent/config/config_file.py index 6838b043..50885556 100644 --- a/src/deadline_worker_agent/config/config_file.py +++ b/src/deadline_worker_agent/config/config_file.py @@ -1,12 +1,18 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. from __future__ import annotations +from dataclasses import dataclass +from enum import Enum from pathlib import Path -from typing import Any, Optional +from typing import Any, NamedTuple, Optional, cast +import shutil import sys import os +import tomlkit from pydantic import BaseModel, BaseSettings, Field, ValidationError, root_validator, StrictStr +from tomlkit.container import Container +from tomlkit.items import Bool, Comment, SingleKey, String, Table, Trivia, Whitespace try: from tomllib import load as load_toml, TOMLDecodeError @@ -15,6 +21,119 @@ from ..capabilities import Capabilities from .errors import ConfigurationError +from . import toml_comment + + +@dataclass(frozen=True) +class ModifiableSettingData: + table_name: str + """The name of the TOML table where the setting is located""" + setting_name: str + """The key for the setting in the TOML table""" + preceding_comment: str + """The comment to be prepended to the setting in the TOML file if the setting did not exist + previously""" + + +class ModifiableSetting(Enum): + ALLOW_EC2_INSTANCE_PROFILE = ModifiableSettingData( + setting_name="allow_ec2_instance_profile", + table_name="aws", + preceding_comment=""" +The "allow_ec2_instance_profile" setting controls whether the Worker will run with an EC2 instance +profile associated to the host instance. This value is overridden when the +DEADLINE_WORKER_ALLOW_INSTANCE_PROFILE environment variable is set using one of the following +case-insensitive values: + + '0', 'off', 'f', 'false', 'n', 'no', '1', 'on', 't', 'true', 'y', 'yes'. + +or if the --disallow-instance-profile command-line flag is specified. + +By default, this value is true and the worker agent will run with or without an instance profile +if the worker is on an EC2 host. If this value is false, the worker host will query the EC2 +instance meta-data service (IMDS) to check for an instance profile. If an instance profile is +detected, the worker agent will stop and exit. + + ***************************************** WARNING ***************************************** + * * + * IF THIS IS TRUE, THEN ANY SESSIONS RUNNING ON THE WORKER CAN ASSUME THE INSTANCE * + * PROFILE AND USE ITS ASSOCIATED PRIVILEGES * + * * + ******************************************************************************************* + +To turn on this feature and have the worker agent not run with an EC2 instance profile, +uncomment the line below: +""".lstrip(), + ) + FARM_ID = ModifiableSettingData( + setting_name="farm_id", + table_name="worker", + preceding_comment=""" +The unique identifier of the AWS Deadline Cloud farm that the Worker belongs to. This value is overridden +when the DEADLINE_WORKER_FARM_ID environment variable is set or the --farm-id command-line argument +is specified. + +The following is an example for setting the farm ID in this configuration file: + +farm_id = "farm-aabbccddeeff11223344556677889900" + +Uncomment the line below and replace the value with your AWS Deadline Cloud farm ID: +""".lstrip(), + ) + FLEET_ID = ModifiableSettingData( + setting_name="fleet_id", + table_name="worker", + preceding_comment=""" +The unique identifier of the AWS Deadline Cloud fleet that the Worker belongs to. This value is overridden +when the DEADLINE_WORKER_FLEET_ID environment variable is set or the --fleet-id command-line +argument is specified. + +The following is an example for setting the fleet ID in this configuration file: + +fleet_id = "fleet-aabbccddeeff11223344556677889900" + +Uncomment the line below and replace the value with your AWS Deadline Cloud fleet ID: +""".lstrip(), + ) + SHUTDOWN_ON_STOP = ModifiableSettingData( + setting_name="shutdown_on_stop", + table_name="os", + preceding_comment=""" +AWS Deadline Cloud may tell the worker to stop. If the "shutdown_on_stop" setting below is true, then the +Worker will attempt to shutdown the host system after the Worker has been stopped. + +This value is overridden when the DEADLINE_WORKER_NO_SHUTDOWN environment variable is set using +one of the following case-insensitive values: + + '0', 'off', 'f', 'false', 'n', 'no', '1', 'on', 't', 'true', 'y', 'yes'. + +or if the --no-shutdown command-line flag is specified. + +To prevent the worker agent from shutting down the host when being told to stop, uncomment the +line below: +""".lstrip(), + ) + WINDOWS_JOB_USER = ModifiableSettingData( + setting_name="windows_job_user", + table_name="os", + preceding_comment=""" +AWS Deadline Cloud may specify a Windows OS user to run a Job's session actions as. Setting +"windows_job_user" will override the OS user and the session actions will be run as +the user given in the value of "windows_job_user" instead. It is important to note that by specifying +this value, the password for the Windows OS user specified will be reset to a random, unstored value. +This setting also requires that the worker agent is run with administrator privileges. This setting is +incompatible the setting "run_jobs_as_agent_user" set to true. + +To have a specific Windows OS user used when running jobs, uncomment the line below and +replace the username as desired. This value is overridden when the DEADLINE_WORKER_WINDOWS_JOB_USER +environment variable or if the --windows-job-user command-line flag is specified. +""".lstrip(), + ) + + +class SettingModification(NamedTuple): + setting: ModifiableSetting + value: str | bool | None # Default path for the Worker configuration file keyed on the value of sys.platform @@ -99,6 +218,162 @@ def get_config_path(cls) -> Path: except KeyError: raise NotImplementedError(f"Unsupported platform {sys.platform}") from None + @classmethod + def backup( + cls, + *, + config_path: Optional[Path] = None, + ) -> None: + if config_path is None: + config_path = cls.get_config_path() + + backup_path = config_path.with_suffix(f"{config_path.suffix}.bak") + shutil.copyfile(config_path, backup_path) + + @classmethod + def modify_settings( + cls, + *, + settings_to_modify: list[SettingModification], + document: tomlkit.TOMLDocument, + ) -> None: + for setting_to_modify in settings_to_modify: + # Lookup the supplementary setting data (section and comment help) + setting_data = setting_to_modify.setting.value + + # Modify the setting, preserving the TOML markup formatting (spaces, tabs, newlines, + # comments, etc..). Also has logic to first attempt to detect commented-out settings, + # if those are found, it uncomments them and sets their value. If a setting is being + # unset, it comments out any existing setting if it existed. + ConfigFile._modify_setting( + document=document, + setting_data=setting_data, + setting_value=setting_to_modify.value, + ) + + @classmethod + def modify_config_file_settings( + cls, + *, + settings_to_modify: list[SettingModification], + backup: bool = False, + config_path: Optional[Path] = None, + ) -> None: + if config_path is None: + config_path = ConfigFile.get_config_path() + + # Load the config document + with config_path.open("r") as f: + document = tomlkit.load(f) + + ConfigFile.modify_settings( + document=document, + settings_to_modify=settings_to_modify, + ) + + if backup: + # Backup the config + ConfigFile.backup(config_path=config_path) + + # Write the modified config + with config_path.open("w") as f: + tomlkit.dump(document, f) + + @staticmethod + def _modify_setting( + *, + document: tomlkit.TOMLDocument, + setting_data: ModifiableSettingData, + setting_value: str | bool | None, + ) -> None: + """Modifies a TOML document to apply the worker agent settings specified in the arguments. + This uses tomlkit to do this while preserving semantics, structure, and style including + spaces, tabs, newlines, and comments). + + If setting_value is None, it assumes the setting should not be specified in the document. + The algorithm is as follows: + + 1. If the document has no setting applied, no action + 1. If the document has an existing setting, it is commented out + + If the setting_value is not None, the algorithm is as follows: + + 1. If the document has an existing setting, the value is changed in place + 2. If the document has a previously commented-out setting, uncomments the last + occurrence and sets its value. + 3. If the document has no prior setting or commented-out setting, a new setting + is added to the bottom of the table with a preceding comment documenting the + setting. + """ + + table_name = setting_data.table_name + setting_name = setting_data.setting_name + preceding_comment = setting_data.preceding_comment + + if table_name not in document: + if setting_value is None: + return + document[table_name] = tomlkit.table() + + target_table = cast(Table, document[table_name]) + key = SingleKey(setting_name) + table_container: Container = target_table.value + + # If there is an existing applied setting in the TOML document + if key in target_table: + # If we are unsetting the value, comment it out. This is to preserve any comments in the + # TOML file (e.g. those originating from the example config file) + if setting_value is None: + toml_comment.comment_out( + table_container=table_container, + key=key, + ) + else: + target_table[setting_name] = setting_value + return + elif setting_value is None: + return + + # Create tomlkit value wrapper object for the value we want to apply. + toml_setting_value: Bool | String + if isinstance(setting_value, str): + toml_setting_value = tomlkit.string(setting_value) + elif isinstance(setting_value, bool): + toml_setting_value = tomlkit.boolean(str(setting_value).lower()) + else: + raise NotImplementedError(f"Unexpected type for setting_value ({type(setting_value)})") + + # Case: There is a commented-out setting (e.g. from the example file) + # We uncomment and set the value. This replicates the original bash implementation behaviour + # of install-deadline-worker + try: + toml_comment.uncomment( + table_container=table_container, + key=key, + value=toml_setting_value, + # We replace the last occurrence since some of the documentation comments include an + # example TOML line before the commented-out line that is supposed to be uncommented + occurrence="last", + ) + except toml_comment.CommentNotFoundError: + pass + else: + return + + # Case: There is no commented-out setting, for example a previous worker agent install where + # the setting did not exist. or a manually created config file + if len(table_container.body) > 0 and not isinstance( + table_container.body[-1][1], Whitespace + ): + target_table.add(tomlkit.nl()) + target_table.add(tomlkit.nl()) + + for line in preceding_comment.splitlines(): + # tomlkit.comment always adds a space after the # + # We remove the space for empty lines + target_table.add(Comment(Trivia(comment_ws=" ", comment=f"# {line}" if line else "#"))) + target_table.add(setting_name, toml_setting_value) + def as_settings( self, settings: BaseSettings, diff --git a/src/deadline_worker_agent/config/toml_comment.py b/src/deadline_worker_agent/config/toml_comment.py new file mode 100644 index 00000000..c1283407 --- /dev/null +++ b/src/deadline_worker_agent/config/toml_comment.py @@ -0,0 +1,81 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +from __future__ import annotations +import re +from typing import Literal + +from tomlkit import comment +from tomlkit.container import Container +from tomlkit.items import ( + Bool, + Comment, + SingleKey, + String, +) + + +class CommentNotFoundError(Exception): + """Raised when trying to comment out a setting that does not exist""" + + def __init__(self, key: SingleKey) -> None: + super(CommentNotFoundError, self).__init__() + self.key = key + + +def comment_out( + *, + table_container: Container, + key: SingleKey, +) -> None: + """Modifies a tomlkit table container to comment out an existing setting""" + + # The _map attribute is a mapping from keys to index (int) / indices (Tuple[int]) in + # the body attribute. This is for efficient lookup into the DOM. + # We pop the entry because we are commenting out the setting. + index = table_container._map.pop(key) + dict.__delitem__(table_container, key.key) + + # index is a tuple for an AOT (array-of-tables). Otherwise an int + assert isinstance(index, int) + + # We get a reference to the old entry. This is a tuple of the form: + # (key, item) + _, prior_value = table_container.body[index] + + # Reference: https://github.com/python-poetry/tomlkit/blob/635831f1be9b0e107047e74af8ebecc7c0e4b7bf/tomlkit/container.py#L569-L577 + # We omit prior_value.trivia.trail since that is a newline + comment_str = ( + f"{key}" + f"{key.sep}" + f"{prior_value.as_string()}" + f"{prior_value.trivia.comment_ws}" + f"{prior_value.trivia.comment}" + ) + table_container.body[index] = (None, comment(comment_str)) + + +def uncomment( + *, + table_container: Container, + key: SingleKey, + value: Bool | String, + occurrence: Literal["first", "last"] = "last", +) -> None: + """Modifies a tomlkit table container to uncomment an existing setting that was previously + commented out.""" + + commented_out_setting = re.compile(f"^#\\s*{re.escape(key.key)}\\s*=.*$") + pos: int | None = None + for i, kv_tuple in enumerate(table_container.body): + table_child_key, table_child_value = kv_tuple + if table_child_key is None and isinstance(table_child_value, Comment): + if commented_out_setting.match(table_child_value.trivia.comment): + pos = i + if occurrence == "first": + break + + if pos is None: + raise CommentNotFoundError(key) + + table_container.body[pos] = (key, value) + table_container._map[key] = pos diff --git a/src/deadline_worker_agent/installer/__init__.py b/src/deadline_worker_agent/installer/__init__.py index 22e0604c..52f1c424 100644 --- a/src/deadline_worker_agent/installer/__init__.py +++ b/src/deadline_worker_agent/installer/__init__.py @@ -122,6 +122,8 @@ def install() -> None: args.user, "--scripts-path", str(scripts_path), + "--python-interpreter-path", + sys.executable, ] if args.vfs_install_path: cmd += ["--vfs-install-path", args.vfs_install_path] diff --git a/src/deadline_worker_agent/installer/install.sh b/src/deadline_worker_agent/installer/install.sh index fb7b90cb..819d1de0 100755 --- a/src/deadline_worker_agent/installer/install.sh +++ b/src/deadline_worker_agent/installer/install.sh @@ -47,13 +47,20 @@ start_service="no" telemetry_opt_out="no" warning_lines=() vfs_install_path="unset" +python_interpreter_path=unset usage() { - echo "Usage: install.sh --farm-id FARM_ID --fleet-id FLEET_ID" - echo " [--region REGION] [--user USER]" - echo " [--scripts-path SCRIPTS_PATH]" + echo "Usage: install.sh --farm-id FARM_ID" + echo " --fleet-id FLEET_ID" + echo " --scripts-path SCRIPTS_PATH" + echo " --python-interpreter-path PYTHON_INTERPRETER_PATH" + echo " --region REGION" + echo " [--user USER]" echo " [-y]" + echo " [--disallow-instance-profile]" + echo " [--no-install-service]" + echo " [--allow-shutdown]" echo " [--vfs-install-path VFS_INSTALL_PATH]" echo "" echo "Arguments" @@ -75,6 +82,9 @@ usage() echo " installed. This is used as the program path when creating the systemd service for the " echo " Worker Agent. If not specified, the first program named 'deadline-worker-agent' and" echo " 'deadline' found in the search path will be used." + echo " --python-interpreter-path" + echo " Path to the Python interpreter for the worker agent. If a Python virtual environment is " + echo " being used, this path should reference the Python executable of the virtual environment." echo " --allow-shutdown" echo " Dictates whether a sudoers rule is created/deleted allowing the worker agent the" echo " ability to shutdown the host system" @@ -121,7 +131,7 @@ validate_deadline_id() { } # Validate arguments -PARSED_ARGUMENTS=$(getopt -n install.sh --longoptions farm-id:,fleet-id:,region:,user:,group:,scripts-path:,vfs-install-path:,start,allow-shutdown,no-install-service,telemetry-opt-out,disallow-instance-profile -- "y" "$@") +PARSED_ARGUMENTS=$(getopt -n install.sh --longoptions farm-id:,fleet-id:,region:,user:,group:,scripts-path:,python-interpreter-path:,vfs-install-path:,start,allow-shutdown,no-install-service,telemetry-opt-out,disallow-instance-profile -- "y" "$@") VALID_ARGUMENTS=$? if [ "${VALID_ARGUMENTS}" != "0" ]; then usage @@ -140,6 +150,7 @@ do --user) wa_user="$2" ; shift 2 ;; --group) job_group="$2" ; shift 2 ;; --scripts-path) scripts_path="$2" ; shift 2 ;; + --python-interpreter-path) python_interpreter_path="$2" ; shift 2 ;; --vfs-install-path) vfs_install_path="$2" ; shift 2 ;; --allow-shutdown) allow_shutdown="yes" ; shift ;; --disallow-instance-profile) disallow_instance_profile="yes" ; shift ;; @@ -173,18 +184,8 @@ elif ! validate_deadline_id fleet "${fleet_id}"; then usage fi if [[ "${scripts_path}" == "unset" ]]; then - set +e - worker_agent_program=$(which deadline-worker-agent) - if [[ "$?" != "0" ]]; then - echo "ERROR: Could not find deadline-worker-agent in search path" - exit 1 - fi - client_library_program=$(which deadline) - if [[ "$?" != "0" ]]; then - echo "ERROR: Could not find deadline in search path" - exit 1 - fi - set -e + echo "ERROR: --scripts-path is not specified" + usage elif [[ ! -d "${scripts_path}" ]]; then echo "ERROR: The specified scripts path is not found: \"${scripts_path}\"" usage @@ -203,6 +204,13 @@ else fi set -e fi +if [[ "${python_interpreter_path}" == "unset" ]]; then + echo "ERROR: --python-interpreter-path is not specified" + usage +elif [[ ! -f "${python_interpreter_path}" ]]; then + echo "ERROR: The Python interpreter path is not found: \"${python_interpreter_path}\"" + usage +fi if [[ "${region}" == "unset" ]]; then echo "ERROR: --region not specified" @@ -394,44 +402,25 @@ echo "Done provisioning configuration directory" if [[ "${allow_shutdown}" == "yes" ]]; then shutdown_on_stop="true" + shutdown_on_stop_flag="--shutdown-on-stop" else shutdown_on_stop="false" + shutdown_on_stop_flag="--no-shutdown-on-stop" fi if [[ "${disallow_instance_profile}" == "yes" ]]; then allow_ec2_instance_profile="false" + allow_ec2_instance_profile_flag="--no-allow-ec2-instance-profile" else allow_ec2_instance_profile="true" + allow_ec2_instance_profile_flag="--allow-ec2-instance-profile" fi -echo "Configuring farm and fleet" -echo "Configuring shutdown on stop" -echo "Configuring allow ec2 instance profile" -sed -E \ - --in-place=.bak \ - -e "s,^# farm_id\s*=\s*\"REPLACE-WITH-WORKER-FARM-ID\"$,farm_id = \"${farm_id}\",g" \ - -e "s,^# fleet_id\s*=\s*\"REPLACE-WITH-WORKER-FLEET-ID\"$,fleet_id = \"${fleet_id}\",g" \ - -e "s,^[#]*\s*shutdown_on_stop\s*=\s*\w+$,shutdown_on_stop = ${shutdown_on_stop},g" \ - -e "s,^[#]*\s*allow_ec2_instance_profile\s*=\s*\w+$,allow_ec2_instance_profile = ${allow_ec2_instance_profile},g" \ - /etc/amazon/deadline/worker.toml -if ! grep "farm_id = \"${farm_id}\"" /etc/amazon/deadline/worker.toml; then - echo "ERROR: Failed to configure farm ID in /etc/amazon/deadline/worker.toml." - exit 1 -fi -if ! grep "fleet_id = \"${fleet_id}\"" /etc/amazon/deadline/worker.toml; then - echo "ERROR: Failed to configure fleet ID in /etc/amazon/deadline/worker.toml." - exit 1 -fi -if ! grep "shutdown_on_stop = ${shutdown_on_stop}" /etc/amazon/deadline/worker.toml; then - echo "ERROR: Failed to configure shutdown on stop in /etc/amazon/deadline/worker.toml." - exit 1 -fi -if ! grep "allow_ec2_instance_profile = ${allow_ec2_instance_profile}" /etc/amazon/deadline/worker.toml; then - echo "ERROR: Failed to configure allow ec2 instance profile in /etc/amazon/deadline/worker.toml." - exit 1 -fi -echo "Done configuring farm and fleet" -echo "Done configuring shutdown on stop" -echo "Done configuring allow ec2 instance profile" +"${python_interpreter_path}" \ + -m deadline_worker_agent.config \ + --farm-id "${farm_id}" \ + --fleet-id "${fleet_id}" \ + "${allow_ec2_instance_profile_flag}" \ + "${shutdown_on_stop_flag}" if ! [[ "${no_install_service}" == "yes" ]]; then # Set up the service diff --git a/src/deadline_worker_agent/installer/win_installer.py b/src/deadline_worker_agent/installer/win_installer.py index 5ab92ab5..f7e172e6 100644 --- a/src/deadline_worker_agent/installer/win_installer.py +++ b/src/deadline_worker_agent/installer/win_installer.py @@ -29,6 +29,11 @@ from openjd.sessions import BadCredentialsException, WindowsSessionUser from win32comext.shell import shell +from ..config.config_file import ( + ConfigFile, + ModifiableSetting, + SettingModification, +) from ..file_system_operations import ( _set_windows_permissions, FileSystemPermissionEnum, @@ -318,123 +323,63 @@ def update_config_file( replaces specific placeholders with the provided values. Parameters: - - deadline_config_sub_directory (str): Subdirectory for Deadline configuration files. - farm_id (str): The farm ID to set in the configuration. - fleet_id (str): The fleet ID to set in the configuration. - shutdown_on_stop (Optional[bool]): The shutdown_on_stop value to set. Does nothing if set to None. + - allow_ec2_instance_profile (Optional[bool]): Whether the agent should be configured to run with[out] an EC2 instance profile. + Does nothing if set to None. + - windows_job_user (Optional[str]): The OS username to be used when running jobs. Overrides the queue's jobRunAs configuration. + Does nothing if set to None. """ logging.info("Updating configuration file") - worker_config_file = os.path.join(deadline_config_sub_directory, "worker.toml") + config_path = Path(deadline_config_sub_directory) / "worker.toml" # Check if the worker.toml file exists, if not, create it from the example - if not os.path.isfile(worker_config_file): + if not os.path.isfile(config_path): # Directory where the script and example configuration files are located. script_dir = os.path.dirname(os.path.realpath(__file__)) example_config_path = os.path.join(script_dir, "worker.toml.example") - shutil.copy(example_config_path, worker_config_file) - - # Make a backup of the worker configuration file - backup_worker_config = worker_config_file + ".bak" - shutil.copy(worker_config_file, backup_worker_config) - - # Read the content of the worker configuration file - with open(worker_config_file, "r") as file: - content = file.read() - - updated_keys = [] - - # Replace the placeholders with actual farm_id and fleet_id - content = re.sub( - r'^# farm_id\s*=\s*("REPLACE-WITH-WORKER-FARM-ID")$', - f'farm_id = "{farm_id}"', - content, - flags=re.MULTILINE, - ) - if not re.search( - rf'^farm_id = "{re.escape(farm_id)}"$', - content, - flags=re.MULTILINE, - ): - raise InstallerFailedException(f"Failed to configure farm ID in {worker_config_file}") - else: - updated_keys.append("farm_id") - content = re.sub( - r'^# fleet_id\s*=\s*("REPLACE-WITH-WORKER-FLEET-ID")$', - f'fleet_id = "{fleet_id}"', - content, - flags=re.MULTILINE, - ) - if not re.search( - rf'^fleet_id = "{re.escape(fleet_id)}"$', - content, - flags=re.MULTILINE, - ): - raise InstallerFailedException(f"Failed to configure fleet ID in {worker_config_file}") - else: - updated_keys.append("fleet_id") + shutil.copy(example_config_path, config_path) + + settings_to_modify: list[SettingModification] = [ + SettingModification( + setting=ModifiableSetting.FARM_ID, + value=farm_id, + ), + SettingModification( + setting=ModifiableSetting.FLEET_ID, + value=fleet_id, + ), + SettingModification( + setting=ModifiableSetting.WINDOWS_JOB_USER, + value=windows_job_user, + ), + ] if shutdown_on_stop is not None: - shutdown_on_stop_toml = str(shutdown_on_stop).lower() - content = re.sub( - r"^#*\s*shutdown_on_stop\s*=\s*\w+$", - f"shutdown_on_stop = {shutdown_on_stop_toml}", - content, - flags=re.MULTILINE, - ) - if not re.search( - rf"^shutdown_on_stop = {re.escape(shutdown_on_stop_toml)}$", - content, - flags=re.MULTILINE, - ): - raise InstallerFailedException( - f"Failed to configure shutdown_on_stop in {worker_config_file}" + settings_to_modify.append( + SettingModification( + setting=ModifiableSetting.SHUTDOWN_ON_STOP, + value=shutdown_on_stop, ) - else: - updated_keys.append("shutdown_on_stop") - if allow_ec2_instance_profile is not None: - allow_ec2_instance_profile_toml = str(allow_ec2_instance_profile).lower() - content = re.sub( - r"^#*\s*allow_ec2_instance_profile\s*=\s*\w+$", - f"allow_ec2_instance_profile = {allow_ec2_instance_profile_toml}", - content, - flags=re.MULTILINE, ) - if not re.search( - rf"^allow_ec2_instance_profile = {re.escape(allow_ec2_instance_profile_toml)}$", - content, - flags=re.MULTILINE, - ): - raise InstallerFailedException( - f"Failed to configure allow_ec2_instance_profile in {worker_config_file}" + if allow_ec2_instance_profile is not None: + settings_to_modify.append( + SettingModification( + setting=ModifiableSetting.ALLOW_EC2_INSTANCE_PROFILE, + value=allow_ec2_instance_profile, ) - else: - updated_keys.append("allow_ec2_instance_profile") - - if windows_job_user is not None: - escaped_username = windows_job_user.replace("\\", "\\\\\\\\") - content = re.sub( - r'^#*\s*windows_job_user\s*=\s*".{1,512}"$', # defer validation to OS - f'windows_job_user = "{escaped_username}"', - content, - flags=re.MULTILINE, ) - search_username = windows_job_user.replace("\\", "\\\\") - if not re.search( - rf'^windows_job_user = "{re.escape(search_username)}"$', - content, - flags=re.MULTILINE, - ): - raise InstallerFailedException( - f"Failed to configure windows_job_user in {worker_config_file}" - ) - else: - updated_keys.append("windows_job_user") - # Write the updated content back to the worker configuration file - with open(worker_config_file, "w") as file: - file.write(content) + updated_keys = [sm.setting.value.setting_name for sm in settings_to_modify] + + ConfigFile.modify_config_file_settings( + settings_to_modify=settings_to_modify, + backup=True, + config_path=config_path, + ) - logging.info(f"Done configuring {updated_keys} in {worker_config_file}") + logging.info(f"Done configuring {updated_keys} in {config_path}") def provision_directories(agent_username: str) -> WorkerAgentDirectories: diff --git a/src/deadline_worker_agent/windows/win_credentials_resolver.py b/src/deadline_worker_agent/windows/win_credentials_resolver.py index 9b657d64..006dae40 100644 --- a/src/deadline_worker_agent/windows/win_credentials_resolver.py +++ b/src/deadline_worker_agent/windows/win_credentials_resolver.py @@ -21,12 +21,12 @@ NoOverflowExponentialBackoff as Backoff, Session as BotoSession, ) -from . import win_service from .win_logon import ( get_windows_credentials, _WindowsCredentialsCacheEntry, unload_and_close, ) +from .win_session import is_windows_session_zero logger = getLogger(__name__) @@ -96,7 +96,7 @@ def _fetch_secret_from_secrets_manager(self, secretArn: str) -> dict: def prune_cache(self): # If we are running as a Windows Service, we maintain a logon token for the user and # do not need to persist the password nor rotate it. - if win_service.is_windows_session_zero(): + if is_windows_session_zero(): return # Filter out entries that haven't been accessed in the last CACHE_EXPIRATION hours @@ -109,7 +109,7 @@ def prune_cache(self): def clear(self): """Clears all users from the cache and cleans up any open resources""" - if win_service.is_windows_session_zero(): + if is_windows_session_zero(): for user in self._user_cache.values(): if user.windows_session_user: logger.info( @@ -131,7 +131,7 @@ def _user_cache_key(*, user_name: str, password_arn: str) -> str: a username and password. For this reason, our cache key should use the password secret ARN since a change of secret may imply a change of password. """ - if win_service.is_windows_session_zero(): + if is_windows_session_zero(): return user_name else: # Create a composite key using user and arn diff --git a/src/deadline_worker_agent/windows/win_logon.py b/src/deadline_worker_agent/windows/win_logon.py index 51872adb..c0bb1b1e 100644 --- a/src/deadline_worker_agent/windows/win_logon.py +++ b/src/deadline_worker_agent/windows/win_logon.py @@ -21,7 +21,8 @@ import secrets import string -from . import win_service + +from .win_session import is_windows_session_zero if TYPE_CHECKING: from _win32typing import PyHKEY, PyHANDLE @@ -69,7 +70,7 @@ def get_windows_credentials(username: str, password: str) -> _WindowsCredentials BadCredentialsException: If the username and/or password are incorrect OSError: If the UserProfile fails to load in session zero """ - if not win_service.is_windows_session_zero(): + if not is_windows_session_zero(): # raises: BadCredentialsException return _WindowsCredentialsCacheEntry( windows_session_user=WindowsSessionUser(user=username, password=password) diff --git a/src/deadline_worker_agent/windows/win_service.py b/src/deadline_worker_agent/windows/win_service.py index f667d2e5..b79f86bc 100644 --- a/src/deadline_worker_agent/windows/win_service.py +++ b/src/deadline_worker_agent/windows/win_service.py @@ -2,13 +2,10 @@ import socket import logging -from functools import cache from threading import Event -import win32process import win32serviceutil import win32service -import win32ts import servicemanager from deadline_worker_agent.startup.entrypoint import entrypoint @@ -60,29 +57,5 @@ def SvcDoRun(self): logger.info("Stop status sent to Windows Service Controller") -def _get_current_process_session() -> int: - """Returns the Windows session ID number for the current process - - Returns - ------- - int - The session ID of the current process - """ - process_id = win32process.GetCurrentProcessId() - return win32ts.ProcessIdToSessionId(process_id) - - -@cache -def is_windows_session_zero() -> bool: - """Returns whether the current Python process is running in Windows session 0. - - Returns - ------- - bool - True if the current process is running in Windows session 0 - """ - return _get_current_process_session() == 0 - - if __name__ == "__main__": win32serviceutil.HandleCommandLine(WorkerAgentWindowsService) diff --git a/src/deadline_worker_agent/windows/win_session.py b/src/deadline_worker_agent/windows/win_session.py new file mode 100644 index 00000000..ea6d3819 --- /dev/null +++ b/src/deadline_worker_agent/windows/win_session.py @@ -0,0 +1,30 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +from functools import cache + +import win32process +import win32ts + + +def _get_current_process_session() -> int: + """Returns the Windows session ID number for the current process + + Returns + ------- + int + The session ID of the current process + """ + process_id = win32process.GetCurrentProcessId() + return win32ts.ProcessIdToSessionId(process_id) + + +@cache +def is_windows_session_zero() -> bool: + """Returns whether the current Python process is running in Windows session 0. + + Returns + ------- + bool + True if the current process is running in Windows session 0 + """ + return _get_current_process_session() == 0 diff --git a/src/deadline_worker_agent/worker.py b/src/deadline_worker_agent/worker.py index 4fb6b307..2e0cc92b 100644 --- a/src/deadline_worker_agent/worker.py +++ b/src/deadline_worker_agent/worker.py @@ -28,6 +28,7 @@ from .scheduler import WorkerScheduler from .sessions import Session + logger = getLogger(__name__) @@ -132,7 +133,7 @@ def __init__( # TODO: Remove this once WA is stable or put behind a debug flag signal.signal(signal.SIGUSR1, self._output_thread_stacks) # type: ignore elif os.name == "nt": - from .windows.win_service import is_windows_session_zero + from .windows.win_session import is_windows_session_zero # If we are in session 0, we are running as a Windows Service using pywin32 # pywin32's pythonservice.exe owns the main thread and the Python application diff --git a/test/integ/installer/__init__.py b/test/integ/config/__init__.py similarity index 100% rename from test/integ/installer/__init__.py rename to test/integ/config/__init__.py diff --git a/test/integ/config/conftest.py b/test/integ/config/conftest.py new file mode 100644 index 00000000..5cbe8e2d --- /dev/null +++ b/test/integ/config/conftest.py @@ -0,0 +1,55 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +from __future__ import annotations +from pathlib import Path + +import pytest + +try: + from tomllib import load as load_toml +except ModuleNotFoundError: + from tomli import load as load_toml + +from deadline_worker_agent.config.config_file import ModifiableSetting + + +TEST_CASE_DATA_BASE_DIR = Path(__file__).parent / "data" +INPUT_CONFIG_FILENAME = "input.toml" +EXPECTED_OUTPUT_CONFIG_FILENAME = "output.toml" + + +@pytest.fixture +def output_path(input_path: str) -> str: + output_path = Path(input_path).parent / EXPECTED_OUTPUT_CONFIG_FILENAME + assert output_path.exists(), f"No matching expected output path for {input_path}" + return str(output_path) + + +@pytest.fixture +def setting_name(input_path: str) -> str: + return Path(input_path).parts[-2] + + +@pytest.fixture +def modifiable_setting(setting_name: str) -> ModifiableSetting: + return getattr(ModifiableSetting, setting_name.upper()) + + +@pytest.fixture +def value_to_set( + setting_name: str, + output_path: str, +) -> str | bool | None: + with open(output_path, "rb") as f: + doc = load_toml(f) + assert len(doc) == 1, f"Only a single section expected, but got {doc}" + for table in doc.values(): + assert isinstance(table, dict) + assert setting_name in table + return table[setting_name] + return None + + +@pytest.fixture +def worker_config_path(tmp_path: Path) -> Path: + return tmp_path / "input.toml" diff --git a/test/integ/config/data/commented/allow_ec2_instance_profile/input.toml b/test/integ/config/data/commented/allow_ec2_instance_profile/input.toml new file mode 100644 index 00000000..a73a1e7b --- /dev/null +++ b/test/integ/config/data/commented/allow_ec2_instance_profile/input.toml @@ -0,0 +1,26 @@ +[aws] + +# The "allow_ec2_instance_profile" setting controls whether the Worker will run with an EC2 instance +# profile associated to the host instance. This value is overridden when the +# DEADLINE_WORKER_ALLOW_INSTANCE_PROFILE environment variable is set using one of the following +# case-insensitive values: +# +# '0', 'off', 'f', 'false', 'n', 'no', '1', 'on', 't', 'true', 'y', 'yes'. +# +# or if the --disallow-instance-profile command-line flag is specified. +# +# By default, this value is true and the worker agent will run with or without an instance profile +# if the worker is on an EC2 host. If this value is false, the worker host will query the EC2 +# instance meta-data service (IMDS) to check for an instance profile. If an instance profile is +# detected, the worker agent will stop and exit. +# +# ***************************************** WARNING ***************************************** +# * * +# * IF THIS IS TRUE, THEN ANY SESSIONS RUNNING ON THE WORKER CAN ASSUME THE INSTANCE * +# * PROFILE AND USE ITS ASSOCIATED PRIVILEGES * +# * * +# ******************************************************************************************* +# +# To turn on this feature and have the worker agent not run with an EC2 instance profile, +# uncomment the line below: +# allow_ec2_instance_profile = false diff --git a/test/integ/config/data/commented/allow_ec2_instance_profile/output.toml b/test/integ/config/data/commented/allow_ec2_instance_profile/output.toml new file mode 100644 index 00000000..070ddedb --- /dev/null +++ b/test/integ/config/data/commented/allow_ec2_instance_profile/output.toml @@ -0,0 +1,26 @@ +[aws] + +# The "allow_ec2_instance_profile" setting controls whether the Worker will run with an EC2 instance +# profile associated to the host instance. This value is overridden when the +# DEADLINE_WORKER_ALLOW_INSTANCE_PROFILE environment variable is set using one of the following +# case-insensitive values: +# +# '0', 'off', 'f', 'false', 'n', 'no', '1', 'on', 't', 'true', 'y', 'yes'. +# +# or if the --disallow-instance-profile command-line flag is specified. +# +# By default, this value is true and the worker agent will run with or without an instance profile +# if the worker is on an EC2 host. If this value is false, the worker host will query the EC2 +# instance meta-data service (IMDS) to check for an instance profile. If an instance profile is +# detected, the worker agent will stop and exit. +# +# ***************************************** WARNING ***************************************** +# * * +# * IF THIS IS TRUE, THEN ANY SESSIONS RUNNING ON THE WORKER CAN ASSUME THE INSTANCE * +# * PROFILE AND USE ITS ASSOCIATED PRIVILEGES * +# * * +# ******************************************************************************************* +# +# To turn on this feature and have the worker agent not run with an EC2 instance profile, +# uncomment the line below: +allow_ec2_instance_profile = true diff --git a/test/integ/config/data/commented/farm_id/input.toml b/test/integ/config/data/commented/farm_id/input.toml new file mode 100644 index 00000000..21ea3402 --- /dev/null +++ b/test/integ/config/data/commented/farm_id/input.toml @@ -0,0 +1,12 @@ +[worker] + +# The unique identifier of the AWS Deadline Cloud farm that the Worker belongs to. This value is overridden +# when the DEADLINE_WORKER_FARM_ID environment variable is set or the --farm-id command-line argument +# is specified. +# +# The following is an example for setting the farm ID in this configuration file: +# +# farm_id = "farm-aabbccddeeff11223344556677889900" +# +# Uncomment the line below and replace the value with your AWS Deadline Cloud farm ID: +# farm_id = "farm-abc" diff --git a/test/integ/config/data/commented/farm_id/output.toml b/test/integ/config/data/commented/farm_id/output.toml new file mode 100644 index 00000000..b906fa73 --- /dev/null +++ b/test/integ/config/data/commented/farm_id/output.toml @@ -0,0 +1,12 @@ +[worker] + +# The unique identifier of the AWS Deadline Cloud farm that the Worker belongs to. This value is overridden +# when the DEADLINE_WORKER_FARM_ID environment variable is set or the --farm-id command-line argument +# is specified. +# +# The following is an example for setting the farm ID in this configuration file: +# +# farm_id = "farm-aabbccddeeff11223344556677889900" +# +# Uncomment the line below and replace the value with your AWS Deadline Cloud farm ID: +farm_id = "farm-123" diff --git a/test/integ/config/data/commented/fleet_id/input.toml b/test/integ/config/data/commented/fleet_id/input.toml new file mode 100644 index 00000000..4f3104e8 --- /dev/null +++ b/test/integ/config/data/commented/fleet_id/input.toml @@ -0,0 +1,12 @@ +[worker] + +# The unique identifier of the AWS Deadline Cloud fleet that the Worker belongs to. This value is overridden +# when the DEADLINE_WORKER_FLEET_ID environment variable is set or the --fleet-id command-line +# argument is specified. +# +# The following is an example for setting the fleet ID in this configuration file: +# +# fleet_id = "fleet-aabbccddeeff11223344556677889900" +# +# Uncomment the line below and replace the value with your AWS Deadline Cloud fleet ID: +# fleet_id = "fleet-input" diff --git a/test/integ/config/data/commented/fleet_id/output.toml b/test/integ/config/data/commented/fleet_id/output.toml new file mode 100644 index 00000000..64714141 --- /dev/null +++ b/test/integ/config/data/commented/fleet_id/output.toml @@ -0,0 +1,12 @@ +[worker] + +# The unique identifier of the AWS Deadline Cloud fleet that the Worker belongs to. This value is overridden +# when the DEADLINE_WORKER_FLEET_ID environment variable is set or the --fleet-id command-line +# argument is specified. +# +# The following is an example for setting the fleet ID in this configuration file: +# +# fleet_id = "fleet-aabbccddeeff11223344556677889900" +# +# Uncomment the line below and replace the value with your AWS Deadline Cloud fleet ID: +fleet_id = "fleet-output" diff --git a/test/integ/config/data/commented/shutdown_on_stop/input.toml b/test/integ/config/data/commented/shutdown_on_stop/input.toml new file mode 100644 index 00000000..fbdb8dbd --- /dev/null +++ b/test/integ/config/data/commented/shutdown_on_stop/input.toml @@ -0,0 +1,15 @@ +[os] + +# AWS Deadline Cloud may tell the worker to stop. If the "shutdown_on_stop" setting below is true, then the +# Worker will attempt to shutdown the host system after the Worker has been stopped. +# +# This value is overridden when the DEADLINE_WORKER_NO_SHUTDOWN environment variable is set using +# one of the following case-insensitive values: +# +# '0', 'off', 'f', 'false', 'n', 'no', '1', 'on', 't', 'true', 'y', 'yes'. +# +# or if the --no-shutdown command-line flag is specified. +# +# To prevent the worker agent from shutting down the host when being told to stop, uncomment the +# line below: +# shutdown_on_stop = false diff --git a/test/integ/config/data/commented/shutdown_on_stop/output.toml b/test/integ/config/data/commented/shutdown_on_stop/output.toml new file mode 100644 index 00000000..888e64ac --- /dev/null +++ b/test/integ/config/data/commented/shutdown_on_stop/output.toml @@ -0,0 +1,15 @@ +[os] + +# AWS Deadline Cloud may tell the worker to stop. If the "shutdown_on_stop" setting below is true, then the +# Worker will attempt to shutdown the host system after the Worker has been stopped. +# +# This value is overridden when the DEADLINE_WORKER_NO_SHUTDOWN environment variable is set using +# one of the following case-insensitive values: +# +# '0', 'off', 'f', 'false', 'n', 'no', '1', 'on', 't', 'true', 'y', 'yes'. +# +# or if the --no-shutdown command-line flag is specified. +# +# To prevent the worker agent from shutting down the host when being told to stop, uncomment the +# line below: +shutdown_on_stop = true diff --git a/test/integ/config/data/commented/windows_job_user/input.toml b/test/integ/config/data/commented/windows_job_user/input.toml new file mode 100644 index 00000000..291f3450 --- /dev/null +++ b/test/integ/config/data/commented/windows_job_user/input.toml @@ -0,0 +1,13 @@ +[os] + +# AWS Deadline Cloud may specify a Windows OS user to run a Job's session actions as. Setting +# "windows_job_user" will override the OS user and the session actions will be run as +# the user given in the value of "windows_job_user" instead. It is important to note that by specifying +# this value, the password for the Windows OS user specified will be reset to a random, unstored value. +# This setting also requires that the worker agent is run with administrator privileges. This setting is +# incompatible the setting "run_jobs_as_agent_user" set to true. +# +# To have a specific Windows OS user used when running jobs, uncomment the line below and +# replace the username as desired. This value is overridden when the DEADLINE_WORKER_WINDOWS_JOB_USER +# environment variable or if the --windows-job-user command-line flag is specified. +# windows_job_user = "input-user" diff --git a/test/integ/config/data/commented/windows_job_user/output.toml b/test/integ/config/data/commented/windows_job_user/output.toml new file mode 100644 index 00000000..c28c965d --- /dev/null +++ b/test/integ/config/data/commented/windows_job_user/output.toml @@ -0,0 +1,13 @@ +[os] + +# AWS Deadline Cloud may specify a Windows OS user to run a Job's session actions as. Setting +# "windows_job_user" will override the OS user and the session actions will be run as +# the user given in the value of "windows_job_user" instead. It is important to note that by specifying +# this value, the password for the Windows OS user specified will be reset to a random, unstored value. +# This setting also requires that the worker agent is run with administrator privileges. This setting is +# incompatible the setting "run_jobs_as_agent_user" set to true. +# +# To have a specific Windows OS user used when running jobs, uncomment the line below and +# replace the username as desired. This value is overridden when the DEADLINE_WORKER_WINDOWS_JOB_USER +# environment variable or if the --windows-job-user command-line flag is specified. +windows_job_user = "output-user" diff --git a/test/integ/config/data/existing/allow_ec2_instance_profile/input.toml b/test/integ/config/data/existing/allow_ec2_instance_profile/input.toml new file mode 100644 index 00000000..38f03be7 --- /dev/null +++ b/test/integ/config/data/existing/allow_ec2_instance_profile/input.toml @@ -0,0 +1,26 @@ +[aws] + +# The "allow_ec2_instance_profile" setting controls whether the Worker will run with an EC2 instance +# profile associated to the host instance. This value is overridden when the +# DEADLINE_WORKER_ALLOW_INSTANCE_PROFILE environment variable is set using one of the following +# case-insensitive values: +# +# '0', 'off', 'f', 'false', 'n', 'no', '1', 'on', 't', 'true', 'y', 'yes'. +# +# or if the --disallow-instance-profile command-line flag is specified. +# +# By default, this value is true and the worker agent will run with or without an instance profile +# if the worker is on an EC2 host. If this value is false, the worker host will query the EC2 +# instance meta-data service (IMDS) to check for an instance profile. If an instance profile is +# detected, the worker agent will stop and exit. +# +# ***************************************** WARNING ***************************************** +# * * +# * IF THIS IS TRUE, THEN ANY SESSIONS RUNNING ON THE WORKER CAN ASSUME THE INSTANCE * +# * PROFILE AND USE ITS ASSOCIATED PRIVILEGES * +# * * +# ******************************************************************************************* +# +# To turn on this feature and have the worker agent not run with an EC2 instance profile, +# uncomment the line below: +allow_ec2_instance_profile = false diff --git a/test/integ/config/data/existing/allow_ec2_instance_profile/output.toml b/test/integ/config/data/existing/allow_ec2_instance_profile/output.toml new file mode 100644 index 00000000..070ddedb --- /dev/null +++ b/test/integ/config/data/existing/allow_ec2_instance_profile/output.toml @@ -0,0 +1,26 @@ +[aws] + +# The "allow_ec2_instance_profile" setting controls whether the Worker will run with an EC2 instance +# profile associated to the host instance. This value is overridden when the +# DEADLINE_WORKER_ALLOW_INSTANCE_PROFILE environment variable is set using one of the following +# case-insensitive values: +# +# '0', 'off', 'f', 'false', 'n', 'no', '1', 'on', 't', 'true', 'y', 'yes'. +# +# or if the --disallow-instance-profile command-line flag is specified. +# +# By default, this value is true and the worker agent will run with or without an instance profile +# if the worker is on an EC2 host. If this value is false, the worker host will query the EC2 +# instance meta-data service (IMDS) to check for an instance profile. If an instance profile is +# detected, the worker agent will stop and exit. +# +# ***************************************** WARNING ***************************************** +# * * +# * IF THIS IS TRUE, THEN ANY SESSIONS RUNNING ON THE WORKER CAN ASSUME THE INSTANCE * +# * PROFILE AND USE ITS ASSOCIATED PRIVILEGES * +# * * +# ******************************************************************************************* +# +# To turn on this feature and have the worker agent not run with an EC2 instance profile, +# uncomment the line below: +allow_ec2_instance_profile = true diff --git a/test/integ/config/data/existing/farm_id/input.toml b/test/integ/config/data/existing/farm_id/input.toml new file mode 100644 index 00000000..cf61d3b7 --- /dev/null +++ b/test/integ/config/data/existing/farm_id/input.toml @@ -0,0 +1,12 @@ +[worker] + +# The unique identifier of the AWS Deadline Cloud farm that the Worker belongs to. This value is overridden +# when the DEADLINE_WORKER_FARM_ID environment variable is set or the --farm-id command-line argument +# is specified. +# +# The following is an example for setting the farm ID in this configuration file: +# +# farm_id = "farm-aabbccddeeff11223344556677889900" +# +# Uncomment the line below and replace the value with your AWS Deadline Cloud farm ID: +farm_id = "farm-abc" diff --git a/test/integ/config/data/existing/farm_id/output.toml b/test/integ/config/data/existing/farm_id/output.toml new file mode 100644 index 00000000..b906fa73 --- /dev/null +++ b/test/integ/config/data/existing/farm_id/output.toml @@ -0,0 +1,12 @@ +[worker] + +# The unique identifier of the AWS Deadline Cloud farm that the Worker belongs to. This value is overridden +# when the DEADLINE_WORKER_FARM_ID environment variable is set or the --farm-id command-line argument +# is specified. +# +# The following is an example for setting the farm ID in this configuration file: +# +# farm_id = "farm-aabbccddeeff11223344556677889900" +# +# Uncomment the line below and replace the value with your AWS Deadline Cloud farm ID: +farm_id = "farm-123" diff --git a/test/integ/config/data/existing/fleet_id/input.toml b/test/integ/config/data/existing/fleet_id/input.toml new file mode 100644 index 00000000..988d58a4 --- /dev/null +++ b/test/integ/config/data/existing/fleet_id/input.toml @@ -0,0 +1,12 @@ +[worker] + +# The unique identifier of the AWS Deadline Cloud fleet that the Worker belongs to. This value is overridden +# when the DEADLINE_WORKER_FLEET_ID environment variable is set or the --fleet-id command-line +# argument is specified. +# +# The following is an example for setting the fleet ID in this configuration file: +# +# fleet_id = "fleet-aabbccddeeff11223344556677889900" +# +# Uncomment the line below and replace the value with your AWS Deadline Cloud fleet ID: +fleet_id = "fleet-input" diff --git a/test/integ/config/data/existing/fleet_id/output.toml b/test/integ/config/data/existing/fleet_id/output.toml new file mode 100644 index 00000000..64714141 --- /dev/null +++ b/test/integ/config/data/existing/fleet_id/output.toml @@ -0,0 +1,12 @@ +[worker] + +# The unique identifier of the AWS Deadline Cloud fleet that the Worker belongs to. This value is overridden +# when the DEADLINE_WORKER_FLEET_ID environment variable is set or the --fleet-id command-line +# argument is specified. +# +# The following is an example for setting the fleet ID in this configuration file: +# +# fleet_id = "fleet-aabbccddeeff11223344556677889900" +# +# Uncomment the line below and replace the value with your AWS Deadline Cloud fleet ID: +fleet_id = "fleet-output" diff --git a/test/integ/config/data/existing/shutdown_on_stop/input.toml b/test/integ/config/data/existing/shutdown_on_stop/input.toml new file mode 100644 index 00000000..c2d3c883 --- /dev/null +++ b/test/integ/config/data/existing/shutdown_on_stop/input.toml @@ -0,0 +1,15 @@ +[os] + +# AWS Deadline Cloud may tell the worker to stop. If the "shutdown_on_stop" setting below is true, then the +# Worker will attempt to shutdown the host system after the Worker has been stopped. +# +# This value is overridden when the DEADLINE_WORKER_NO_SHUTDOWN environment variable is set using +# one of the following case-insensitive values: +# +# '0', 'off', 'f', 'false', 'n', 'no', '1', 'on', 't', 'true', 'y', 'yes'. +# +# or if the --no-shutdown command-line flag is specified. +# +# To prevent the worker agent from shutting down the host when being told to stop, uncomment the +# line below: +shutdown_on_stop = false diff --git a/test/integ/config/data/existing/shutdown_on_stop/output.toml b/test/integ/config/data/existing/shutdown_on_stop/output.toml new file mode 100644 index 00000000..888e64ac --- /dev/null +++ b/test/integ/config/data/existing/shutdown_on_stop/output.toml @@ -0,0 +1,15 @@ +[os] + +# AWS Deadline Cloud may tell the worker to stop. If the "shutdown_on_stop" setting below is true, then the +# Worker will attempt to shutdown the host system after the Worker has been stopped. +# +# This value is overridden when the DEADLINE_WORKER_NO_SHUTDOWN environment variable is set using +# one of the following case-insensitive values: +# +# '0', 'off', 'f', 'false', 'n', 'no', '1', 'on', 't', 'true', 'y', 'yes'. +# +# or if the --no-shutdown command-line flag is specified. +# +# To prevent the worker agent from shutting down the host when being told to stop, uncomment the +# line below: +shutdown_on_stop = true diff --git a/test/integ/config/data/existing/windows_job_user/input.toml b/test/integ/config/data/existing/windows_job_user/input.toml new file mode 100644 index 00000000..a1dbb8a9 --- /dev/null +++ b/test/integ/config/data/existing/windows_job_user/input.toml @@ -0,0 +1,13 @@ +[os] + +# AWS Deadline Cloud may specify a Windows OS user to run a Job's session actions as. Setting +# "windows_job_user" will override the OS user and the session actions will be run as +# the user given in the value of "windows_job_user" instead. It is important to note that by specifying +# this value, the password for the Windows OS user specified will be reset to a random, unstored value. +# This setting also requires that the worker agent is run with administrator privileges. This setting is +# incompatible the setting "run_jobs_as_agent_user" set to true. +# +# To have a specific Windows OS user used when running jobs, uncomment the line below and +# replace the username as desired. This value is overridden when the DEADLINE_WORKER_WINDOWS_JOB_USER +# environment variable or if the --windows-job-user command-line flag is specified. +windows_job_user = "input-user" diff --git a/test/integ/config/data/existing/windows_job_user/output.toml b/test/integ/config/data/existing/windows_job_user/output.toml new file mode 100644 index 00000000..c28c965d --- /dev/null +++ b/test/integ/config/data/existing/windows_job_user/output.toml @@ -0,0 +1,13 @@ +[os] + +# AWS Deadline Cloud may specify a Windows OS user to run a Job's session actions as. Setting +# "windows_job_user" will override the OS user and the session actions will be run as +# the user given in the value of "windows_job_user" instead. It is important to note that by specifying +# this value, the password for the Windows OS user specified will be reset to a random, unstored value. +# This setting also requires that the worker agent is run with administrator privileges. This setting is +# incompatible the setting "run_jobs_as_agent_user" set to true. +# +# To have a specific Windows OS user used when running jobs, uncomment the line below and +# replace the username as desired. This value is overridden when the DEADLINE_WORKER_WINDOWS_JOB_USER +# environment variable or if the --windows-job-user command-line flag is specified. +windows_job_user = "output-user" diff --git a/test/integ/config/data/missing/allow_ec2_instance_profile/input.toml b/test/integ/config/data/missing/allow_ec2_instance_profile/input.toml new file mode 100644 index 00000000..f08a053d --- /dev/null +++ b/test/integ/config/data/missing/allow_ec2_instance_profile/input.toml @@ -0,0 +1,7 @@ +[aws] + +# Some prior content that should be preserved +prior_setting_1 = 1 + +# More content to preserve +prior_setting_2 = "abc" # and here's an inline comment \ No newline at end of file diff --git a/test/integ/config/data/missing/allow_ec2_instance_profile/output.toml b/test/integ/config/data/missing/allow_ec2_instance_profile/output.toml new file mode 100644 index 00000000..cea411ab --- /dev/null +++ b/test/integ/config/data/missing/allow_ec2_instance_profile/output.toml @@ -0,0 +1,32 @@ +[aws] + +# Some prior content that should be preserved +prior_setting_1 = 1 + +# More content to preserve +prior_setting_2 = "abc" # and here's an inline comment + +# The "allow_ec2_instance_profile" setting controls whether the Worker will run with an EC2 instance +# profile associated to the host instance. This value is overridden when the +# DEADLINE_WORKER_ALLOW_INSTANCE_PROFILE environment variable is set using one of the following +# case-insensitive values: +# +# '0', 'off', 'f', 'false', 'n', 'no', '1', 'on', 't', 'true', 'y', 'yes'. +# +# or if the --disallow-instance-profile command-line flag is specified. +# +# By default, this value is true and the worker agent will run with or without an instance profile +# if the worker is on an EC2 host. If this value is false, the worker host will query the EC2 +# instance meta-data service (IMDS) to check for an instance profile. If an instance profile is +# detected, the worker agent will stop and exit. +# +# ***************************************** WARNING ***************************************** +# * * +# * IF THIS IS TRUE, THEN ANY SESSIONS RUNNING ON THE WORKER CAN ASSUME THE INSTANCE * +# * PROFILE AND USE ITS ASSOCIATED PRIVILEGES * +# * * +# ******************************************************************************************* +# +# To turn on this feature and have the worker agent not run with an EC2 instance profile, +# uncomment the line below: +allow_ec2_instance_profile = false diff --git a/test/integ/config/data/missing/farm_id/input.toml b/test/integ/config/data/missing/farm_id/input.toml new file mode 100644 index 00000000..05d64eff --- /dev/null +++ b/test/integ/config/data/missing/farm_id/input.toml @@ -0,0 +1,7 @@ +[worker] + +# Some prior content that should be preserved +prior_setting_1 = 1 + +# More content to preserve +prior_setting_2 = "abc" # and here's an inline comment \ No newline at end of file diff --git a/test/integ/config/data/missing/farm_id/output.toml b/test/integ/config/data/missing/farm_id/output.toml new file mode 100644 index 00000000..edd5d127 --- /dev/null +++ b/test/integ/config/data/missing/farm_id/output.toml @@ -0,0 +1,18 @@ +[worker] + +# Some prior content that should be preserved +prior_setting_1 = 1 + +# More content to preserve +prior_setting_2 = "abc" # and here's an inline comment + +# The unique identifier of the AWS Deadline Cloud farm that the Worker belongs to. This value is overridden +# when the DEADLINE_WORKER_FARM_ID environment variable is set or the --farm-id command-line argument +# is specified. +# +# The following is an example for setting the farm ID in this configuration file: +# +# farm_id = "farm-aabbccddeeff11223344556677889900" +# +# Uncomment the line below and replace the value with your AWS Deadline Cloud farm ID: +farm_id = "farm-123" diff --git a/test/integ/config/data/missing/fleet_id/input.toml b/test/integ/config/data/missing/fleet_id/input.toml new file mode 100644 index 00000000..05d64eff --- /dev/null +++ b/test/integ/config/data/missing/fleet_id/input.toml @@ -0,0 +1,7 @@ +[worker] + +# Some prior content that should be preserved +prior_setting_1 = 1 + +# More content to preserve +prior_setting_2 = "abc" # and here's an inline comment \ No newline at end of file diff --git a/test/integ/config/data/missing/fleet_id/output.toml b/test/integ/config/data/missing/fleet_id/output.toml new file mode 100644 index 00000000..1389caef --- /dev/null +++ b/test/integ/config/data/missing/fleet_id/output.toml @@ -0,0 +1,18 @@ +[worker] + +# Some prior content that should be preserved +prior_setting_1 = 1 + +# More content to preserve +prior_setting_2 = "abc" # and here's an inline comment + +# The unique identifier of the AWS Deadline Cloud fleet that the Worker belongs to. This value is overridden +# when the DEADLINE_WORKER_FLEET_ID environment variable is set or the --fleet-id command-line +# argument is specified. +# +# The following is an example for setting the fleet ID in this configuration file: +# +# fleet_id = "fleet-aabbccddeeff11223344556677889900" +# +# Uncomment the line below and replace the value with your AWS Deadline Cloud fleet ID: +fleet_id = "fleet-abc" diff --git a/test/integ/config/data/missing/shutdown_on_stop/input.toml b/test/integ/config/data/missing/shutdown_on_stop/input.toml new file mode 100644 index 00000000..0f3d6e35 --- /dev/null +++ b/test/integ/config/data/missing/shutdown_on_stop/input.toml @@ -0,0 +1,7 @@ +[os] + +# Some prior content that should be preserved +prior_setting_1 = 1 + +# More content to preserve +prior_setting_2 = "abc" # and here's an inline comment \ No newline at end of file diff --git a/test/integ/config/data/missing/shutdown_on_stop/output.toml b/test/integ/config/data/missing/shutdown_on_stop/output.toml new file mode 100644 index 00000000..a21523e9 --- /dev/null +++ b/test/integ/config/data/missing/shutdown_on_stop/output.toml @@ -0,0 +1,21 @@ +[os] + +# Some prior content that should be preserved +prior_setting_1 = 1 + +# More content to preserve +prior_setting_2 = "abc" # and here's an inline comment + +# AWS Deadline Cloud may tell the worker to stop. If the "shutdown_on_stop" setting below is true, then the +# Worker will attempt to shutdown the host system after the Worker has been stopped. +# +# This value is overridden when the DEADLINE_WORKER_NO_SHUTDOWN environment variable is set using +# one of the following case-insensitive values: +# +# '0', 'off', 'f', 'false', 'n', 'no', '1', 'on', 't', 'true', 'y', 'yes'. +# +# or if the --no-shutdown command-line flag is specified. +# +# To prevent the worker agent from shutting down the host when being told to stop, uncomment the +# line below: +shutdown_on_stop = true diff --git a/test/integ/config/data/missing/windows_job_user/input.toml b/test/integ/config/data/missing/windows_job_user/input.toml new file mode 100644 index 00000000..0f3d6e35 --- /dev/null +++ b/test/integ/config/data/missing/windows_job_user/input.toml @@ -0,0 +1,7 @@ +[os] + +# Some prior content that should be preserved +prior_setting_1 = 1 + +# More content to preserve +prior_setting_2 = "abc" # and here's an inline comment \ No newline at end of file diff --git a/test/integ/config/data/missing/windows_job_user/output.toml b/test/integ/config/data/missing/windows_job_user/output.toml new file mode 100644 index 00000000..3f301f5f --- /dev/null +++ b/test/integ/config/data/missing/windows_job_user/output.toml @@ -0,0 +1,19 @@ +[os] + +# Some prior content that should be preserved +prior_setting_1 = 1 + +# More content to preserve +prior_setting_2 = "abc" # and here's an inline comment + +# AWS Deadline Cloud may specify a Windows OS user to run a Job's session actions as. Setting +# "windows_job_user" will override the OS user and the session actions will be run as +# the user given in the value of "windows_job_user" instead. It is important to note that by specifying +# this value, the password for the Windows OS user specified will be reset to a random, unstored value. +# This setting also requires that the worker agent is run with administrator privileges. This setting is +# incompatible the setting "run_jobs_as_agent_user" set to true. +# +# To have a specific Windows OS user used when running jobs, uncomment the line below and +# replace the username as desired. This value is overridden when the DEADLINE_WORKER_WINDOWS_JOB_USER +# environment variable or if the --windows-job-user command-line flag is specified. +windows_job_user = "job-user" diff --git a/test/integ/config/data/unset/allow_ec2_instance_profile/input.toml b/test/integ/config/data/unset/allow_ec2_instance_profile/input.toml new file mode 100644 index 00000000..38f03be7 --- /dev/null +++ b/test/integ/config/data/unset/allow_ec2_instance_profile/input.toml @@ -0,0 +1,26 @@ +[aws] + +# The "allow_ec2_instance_profile" setting controls whether the Worker will run with an EC2 instance +# profile associated to the host instance. This value is overridden when the +# DEADLINE_WORKER_ALLOW_INSTANCE_PROFILE environment variable is set using one of the following +# case-insensitive values: +# +# '0', 'off', 'f', 'false', 'n', 'no', '1', 'on', 't', 'true', 'y', 'yes'. +# +# or if the --disallow-instance-profile command-line flag is specified. +# +# By default, this value is true and the worker agent will run with or without an instance profile +# if the worker is on an EC2 host. If this value is false, the worker host will query the EC2 +# instance meta-data service (IMDS) to check for an instance profile. If an instance profile is +# detected, the worker agent will stop and exit. +# +# ***************************************** WARNING ***************************************** +# * * +# * IF THIS IS TRUE, THEN ANY SESSIONS RUNNING ON THE WORKER CAN ASSUME THE INSTANCE * +# * PROFILE AND USE ITS ASSOCIATED PRIVILEGES * +# * * +# ******************************************************************************************* +# +# To turn on this feature and have the worker agent not run with an EC2 instance profile, +# uncomment the line below: +allow_ec2_instance_profile = false diff --git a/test/integ/config/data/unset/allow_ec2_instance_profile/output.toml b/test/integ/config/data/unset/allow_ec2_instance_profile/output.toml new file mode 100644 index 00000000..a73a1e7b --- /dev/null +++ b/test/integ/config/data/unset/allow_ec2_instance_profile/output.toml @@ -0,0 +1,26 @@ +[aws] + +# The "allow_ec2_instance_profile" setting controls whether the Worker will run with an EC2 instance +# profile associated to the host instance. This value is overridden when the +# DEADLINE_WORKER_ALLOW_INSTANCE_PROFILE environment variable is set using one of the following +# case-insensitive values: +# +# '0', 'off', 'f', 'false', 'n', 'no', '1', 'on', 't', 'true', 'y', 'yes'. +# +# or if the --disallow-instance-profile command-line flag is specified. +# +# By default, this value is true and the worker agent will run with or without an instance profile +# if the worker is on an EC2 host. If this value is false, the worker host will query the EC2 +# instance meta-data service (IMDS) to check for an instance profile. If an instance profile is +# detected, the worker agent will stop and exit. +# +# ***************************************** WARNING ***************************************** +# * * +# * IF THIS IS TRUE, THEN ANY SESSIONS RUNNING ON THE WORKER CAN ASSUME THE INSTANCE * +# * PROFILE AND USE ITS ASSOCIATED PRIVILEGES * +# * * +# ******************************************************************************************* +# +# To turn on this feature and have the worker agent not run with an EC2 instance profile, +# uncomment the line below: +# allow_ec2_instance_profile = false diff --git a/test/integ/config/data/unset/farm_id/input.toml b/test/integ/config/data/unset/farm_id/input.toml new file mode 100644 index 00000000..4624c0c0 --- /dev/null +++ b/test/integ/config/data/unset/farm_id/input.toml @@ -0,0 +1,12 @@ +[worker] + +# The unique identifier of the AWS Deadline Cloud farm that the Worker belongs to. This value is overridden +# when the DEADLINE_WORKER_FARM_ID environment variable is set or the --farm-id command-line argument +# is specified. +# +# The following is an example for setting the farm ID in this configuration file: +# +# farm_id = "farm-aabbccddeeff11223344556677889900" +# +# Uncomment the line below and replace the value with your AWS Deadline Cloud farm ID: +farm_id = "farm-input" diff --git a/test/integ/config/data/unset/farm_id/output.toml b/test/integ/config/data/unset/farm_id/output.toml new file mode 100644 index 00000000..81529d0b --- /dev/null +++ b/test/integ/config/data/unset/farm_id/output.toml @@ -0,0 +1,12 @@ +[worker] + +# The unique identifier of the AWS Deadline Cloud farm that the Worker belongs to. This value is overridden +# when the DEADLINE_WORKER_FARM_ID environment variable is set or the --farm-id command-line argument +# is specified. +# +# The following is an example for setting the farm ID in this configuration file: +# +# farm_id = "farm-aabbccddeeff11223344556677889900" +# +# Uncomment the line below and replace the value with your AWS Deadline Cloud farm ID: +# farm_id = "farm-input" diff --git a/test/integ/config/data/unset/fleet_id/input.toml b/test/integ/config/data/unset/fleet_id/input.toml new file mode 100644 index 00000000..988d58a4 --- /dev/null +++ b/test/integ/config/data/unset/fleet_id/input.toml @@ -0,0 +1,12 @@ +[worker] + +# The unique identifier of the AWS Deadline Cloud fleet that the Worker belongs to. This value is overridden +# when the DEADLINE_WORKER_FLEET_ID environment variable is set or the --fleet-id command-line +# argument is specified. +# +# The following is an example for setting the fleet ID in this configuration file: +# +# fleet_id = "fleet-aabbccddeeff11223344556677889900" +# +# Uncomment the line below and replace the value with your AWS Deadline Cloud fleet ID: +fleet_id = "fleet-input" diff --git a/test/integ/config/data/unset/fleet_id/output.toml b/test/integ/config/data/unset/fleet_id/output.toml new file mode 100644 index 00000000..4f3104e8 --- /dev/null +++ b/test/integ/config/data/unset/fleet_id/output.toml @@ -0,0 +1,12 @@ +[worker] + +# The unique identifier of the AWS Deadline Cloud fleet that the Worker belongs to. This value is overridden +# when the DEADLINE_WORKER_FLEET_ID environment variable is set or the --fleet-id command-line +# argument is specified. +# +# The following is an example for setting the fleet ID in this configuration file: +# +# fleet_id = "fleet-aabbccddeeff11223344556677889900" +# +# Uncomment the line below and replace the value with your AWS Deadline Cloud fleet ID: +# fleet_id = "fleet-input" diff --git a/test/integ/config/data/unset/shutdown_on_stop/input.toml b/test/integ/config/data/unset/shutdown_on_stop/input.toml new file mode 100644 index 00000000..c2d3c883 --- /dev/null +++ b/test/integ/config/data/unset/shutdown_on_stop/input.toml @@ -0,0 +1,15 @@ +[os] + +# AWS Deadline Cloud may tell the worker to stop. If the "shutdown_on_stop" setting below is true, then the +# Worker will attempt to shutdown the host system after the Worker has been stopped. +# +# This value is overridden when the DEADLINE_WORKER_NO_SHUTDOWN environment variable is set using +# one of the following case-insensitive values: +# +# '0', 'off', 'f', 'false', 'n', 'no', '1', 'on', 't', 'true', 'y', 'yes'. +# +# or if the --no-shutdown command-line flag is specified. +# +# To prevent the worker agent from shutting down the host when being told to stop, uncomment the +# line below: +shutdown_on_stop = false diff --git a/test/integ/config/data/unset/shutdown_on_stop/output.toml b/test/integ/config/data/unset/shutdown_on_stop/output.toml new file mode 100644 index 00000000..fbdb8dbd --- /dev/null +++ b/test/integ/config/data/unset/shutdown_on_stop/output.toml @@ -0,0 +1,15 @@ +[os] + +# AWS Deadline Cloud may tell the worker to stop. If the "shutdown_on_stop" setting below is true, then the +# Worker will attempt to shutdown the host system after the Worker has been stopped. +# +# This value is overridden when the DEADLINE_WORKER_NO_SHUTDOWN environment variable is set using +# one of the following case-insensitive values: +# +# '0', 'off', 'f', 'false', 'n', 'no', '1', 'on', 't', 'true', 'y', 'yes'. +# +# or if the --no-shutdown command-line flag is specified. +# +# To prevent the worker agent from shutting down the host when being told to stop, uncomment the +# line below: +# shutdown_on_stop = false diff --git a/test/integ/config/data/unset/windows_job_user/input.toml b/test/integ/config/data/unset/windows_job_user/input.toml new file mode 100644 index 00000000..a1dbb8a9 --- /dev/null +++ b/test/integ/config/data/unset/windows_job_user/input.toml @@ -0,0 +1,13 @@ +[os] + +# AWS Deadline Cloud may specify a Windows OS user to run a Job's session actions as. Setting +# "windows_job_user" will override the OS user and the session actions will be run as +# the user given in the value of "windows_job_user" instead. It is important to note that by specifying +# this value, the password for the Windows OS user specified will be reset to a random, unstored value. +# This setting also requires that the worker agent is run with administrator privileges. This setting is +# incompatible the setting "run_jobs_as_agent_user" set to true. +# +# To have a specific Windows OS user used when running jobs, uncomment the line below and +# replace the username as desired. This value is overridden when the DEADLINE_WORKER_WINDOWS_JOB_USER +# environment variable or if the --windows-job-user command-line flag is specified. +windows_job_user = "input-user" diff --git a/test/integ/config/data/unset/windows_job_user/output.toml b/test/integ/config/data/unset/windows_job_user/output.toml new file mode 100644 index 00000000..291f3450 --- /dev/null +++ b/test/integ/config/data/unset/windows_job_user/output.toml @@ -0,0 +1,13 @@ +[os] + +# AWS Deadline Cloud may specify a Windows OS user to run a Job's session actions as. Setting +# "windows_job_user" will override the OS user and the session actions will be run as +# the user given in the value of "windows_job_user" instead. It is important to note that by specifying +# this value, the password for the Windows OS user specified will be reset to a random, unstored value. +# This setting also requires that the worker agent is run with administrator privileges. This setting is +# incompatible the setting "run_jobs_as_agent_user" set to true. +# +# To have a specific Windows OS user used when running jobs, uncomment the line below and +# replace the username as desired. This value is overridden when the DEADLINE_WORKER_WINDOWS_JOB_USER +# environment variable or if the --windows-job-user command-line flag is specified. +# windows_job_user = "input-user" diff --git a/test/integ/config/test_config_cli.py b/test/integ/config/test_config_cli.py new file mode 100644 index 00000000..c650a7f0 --- /dev/null +++ b/test/integ/config/test_config_cli.py @@ -0,0 +1,369 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +""" +This test module contains end-to-end tests that cover using deadline_worker_agent.config module +using its command-line interface +""" + +from __future__ import annotations +import glob +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Callable + +import pytest + +from deadline_worker_agent.config import config_file + +from .conftest import ( + TEST_CASE_DATA_BASE_DIR, + INPUT_CONFIG_FILENAME, +) + + +def cli_args_for_allow_ec2_instance_profile(value: str | bool | None) -> list[str]: + if value is None: + return [] + elif isinstance(value, bool): + return ["--allow-ec2-instance-profile" if value else "--no-allow-ec2-instance-profile"] + else: + raise NotImplementedError(f"Unexpected value: {value}") + + +def cli_args_for_farm_id(value: str | bool | None) -> list[str]: + if value is None: + return [] + elif isinstance(value, str): + return ["--farm-id", value] + else: + raise NotImplementedError(f"Unexpected value: {value}") + + +def cli_args_for_fleet_id(value: str | bool | None) -> list[str]: + if value is None: + return [] + elif isinstance(value, str): + return ["--fleet-id", value] + else: + raise NotImplementedError(f"Unexpected value: {value}") + + +def cli_args_for_windows_job_user(value: str | bool | None) -> list[str]: + if value is None: + return ["--no-windows-job-user"] + elif isinstance(value, str): + return ["--windows-job-user", value] + else: + raise NotImplementedError(f"Unexpected value: {value}") + + +def cli_args_for_shutdown_on_stop(value: str | bool | None) -> list[str]: + if value is None: + return [] + elif value is True: + return ["--shutdown-on-stop"] + elif value is False: + return ["--no-shutdown-on-stop"] + else: + raise NotImplementedError(f"Unexpected value: {value}") + + +SETTING_TO_CLI_ARGS: dict[ + config_file.ModifiableSetting, Callable[[str | bool | None], list[str]] +] = { + config_file.ModifiableSetting.ALLOW_EC2_INSTANCE_PROFILE: cli_args_for_allow_ec2_instance_profile, + config_file.ModifiableSetting.FARM_ID: cli_args_for_farm_id, + config_file.ModifiableSetting.FLEET_ID: cli_args_for_fleet_id, + config_file.ModifiableSetting.WINDOWS_JOB_USER: cli_args_for_windows_job_user, + config_file.ModifiableSetting.SHUTDOWN_ON_STOP: cli_args_for_shutdown_on_stop, +} + + +@pytest.fixture +def value_to_set_cli_args( + modifiable_setting: config_file.ModifiableSetting, + value_to_set: str | bool | None, +) -> list[str]: + try: + setting_to_cli_args = SETTING_TO_CLI_ARGS[modifiable_setting] + except KeyError: + raise NotImplementedError(f"Unhandled setting: {modifiable_setting.name}") from None + + return setting_to_cli_args(value_to_set) + + +class TestMissingFromInput: + """Tests that when an there is no existing setting or commented-out setting in the TOML file + and a new value is provided, that the setting is added to the file along with TOML + comments that document the setting. All existing content in the TOML document including + comments and blank space should be preserved. + + This relies on conventional directory structure relative to the parent directory of this + test module: + + data/ + unset/ + / + input.toml + output.toml + + Where is the lower-case match of each enum value in + deadline_worker_agent.config.config_file.ModifiableSetting. + + The input.toml file serves as the existing input config file. + + The output.toml is the expected output file to be generated. The expectation is that the + setting denoted by in output.toml contains the output value to be applied + to input.toml. This value is obtained from the output file using tomllib/tomli and then used + to modify the input TOML document. + """ + + INPUT_FILES = glob.glob(str(TEST_CASE_DATA_BASE_DIR / "missing" / "*" / INPUT_CONFIG_FILENAME)) + + @pytest.fixture(params=INPUT_FILES, ids=[Path(f).parts[-2] for f in INPUT_FILES]) + def input_path(self, request: pytest.FixtureRequest) -> str: + return request.param + + def test( + self, + input_path: str, + output_path: str, + worker_config_path: Path, + value_to_set_cli_args: list[str], + ) -> None: + # GIVEN + with open(output_path, "r") as f: + expected_output = f.read() + shutil.copyfile(input_path, worker_config_path) + cmd = [ + sys.executable, + "-m", + "deadline_worker_agent.config", + "--config-path", + str(worker_config_path), + *value_to_set_cli_args, + ] + + # WHEN + subprocess.run(cmd, check=True) + + # THEN + written_config = worker_config_path.read_text() + assert expected_output == written_config + + def test_coverage( + self, + ) -> None: + """Tests that we have coverage for all of the possible ModifiableSetting values""" + # GIVEN + attributes_covered = {Path(f).parts[-2].upper() for f in self.INPUT_FILES} + attributes = set(config_file.ModifiableSetting._member_names_) + + # THEN + assert attributes_covered == attributes + + +class TestModifyExistingSettingInInput: + """Tests that when an there is an existing active setting in the TOML file and a new value, + is provided, that the setting is updated and all TOML comments and blank space are + preserved. + + This relies on conventional directory structure relative to the parent directory of this + test module: + + data/ + unset/ + / + input.toml + output.toml + + Where is the lower-case match of each enum value in + deadline_worker_agent.config.config_file.ModifiableSetting. + + The input.toml file serves as the existing input config file. + + The output.toml is the expected output file to be generated. The expectation is that the + setting denoted by in output.toml has a modified value from input.toml. This + value is obtained from the output file using tomllib/tomli and then used to modify the + input TOML document. + """ + + INPUT_FILES = glob.glob(str(TEST_CASE_DATA_BASE_DIR / "existing" / "*" / INPUT_CONFIG_FILENAME)) + + @pytest.fixture(params=INPUT_FILES, ids=[Path(f).parts[-2] for f in INPUT_FILES]) + def input_path(self, request: pytest.FixtureRequest) -> str: + return request.param + + def test( + self, + input_path: str, + output_path: str, + worker_config_path: Path, + value_to_set_cli_args: list[str], + ) -> None: + # GIVEN + with open(output_path, "r") as f: + expected_output = f.read() + shutil.copyfile(input_path, worker_config_path) + cmd = [ + sys.executable, + "-m", + "deadline_worker_agent.config", + "--config-path", + str(worker_config_path), + *value_to_set_cli_args, + ] + + # WHEN + subprocess.run(cmd, check=True) + + # THEN + written_config = worker_config_path.read_text() + assert expected_output == written_config + + def test_coverage( + self, + ) -> None: + """Tests that we have coverage for all of the possible ModifiableSetting values""" + # GIVEN + attributes_covered = {Path(f).parts[-2].upper() for f in self.INPUT_FILES} + attributes = set(config_file.ModifiableSetting._member_names_) + + # THEN + assert attributes_covered == attributes + + +class TestSettingValueUncommentsCommentedOutSetting: + """Tests that when an there is an existing setting in the TOML file that is commented out + and a new value is provided, that the commented-out setting becomes un-commented and its + value is updated. + + This relies on conventional directory structure relative to the parent directory of this + test module: + + data/ + unset/ + / + input.toml + output.toml + + Where is the lower-case match of each enum value in + deadline_worker_agent.config.config_file.ModifiableSetting. + + The input.toml file serves as the existing input config file. + + The output.toml is the expected output file to be generated. The expectation is that the + setting denoted by in output.toml has a modified value for the corresponding + commented-out setting in input.toml. This value is obtained from the output file using + tomllib/tomli and then used to modify the input TOML document. + """ + + INPUT_FILES = glob.glob( + str(TEST_CASE_DATA_BASE_DIR / "commented" / "*" / INPUT_CONFIG_FILENAME) + ) + + @pytest.fixture(params=INPUT_FILES, ids=[Path(f).parts[-2] for f in INPUT_FILES]) + def input_path(self, request: pytest.FixtureRequest) -> str: + return request.param + + def test( + self, + input_path: str, + output_path: str, + worker_config_path: Path, + value_to_set_cli_args: list[str], + ) -> None: + # GIVEN + with open(output_path, "r") as f: + expected_output = f.read() + shutil.copyfile(input_path, worker_config_path) + cmd = [ + sys.executable, + "-m", + "deadline_worker_agent.config", + "--config-path", + str(worker_config_path), + *value_to_set_cli_args, + ] + + # WHEN + subprocess.run(cmd, check=True) + + # THEN + written_config = worker_config_path.read_text() + assert expected_output == written_config + + def test_coverage( + self, + ) -> None: + """Tests that we have coverage for all of the possible ModifiableSetting values""" + # GIVEN + attributes_covered = {Path(f).parts[-2].upper() for f in self.INPUT_FILES} + attributes = set(config_file.ModifiableSetting._member_names_) + + # THEN + assert attributes_covered == attributes + + +class TestUnsetCommentsOutInputSetting: + """Tests that when an there is an existing active setting in the TOML file that is being + unset, that the setting becomes commented out and all TOML comments and blank space are + preserved. + + This relies on conventional directory structure relative to the parent directory of this + test module: + + data/ + unset/ + / + input.toml + output.toml + + Where is the lower-case match of each enum value in + deadline_worker_agent.config.config_file.ModifiableSetting. + + The input.toml file serves as the existing input config file. The output.toml is the + expected output file to be generated with the setting denoted by commented + out. + """ + + @pytest.fixture + def input_path(self) -> str: + return str(TEST_CASE_DATA_BASE_DIR / "unset" / "windows_job_user" / "input.toml") + + @pytest.fixture + def value_to_set_cli_args(self) -> list[str]: + return ["--no-windows-job-user"] + + def test( + self, + input_path: str, + output_path: str, + setting_name: str, + worker_config_path: Path, + ) -> None: + """The functional test. See class docstring""" + # GIVEN + value_to_set = None + modifiable_setting = getattr(config_file.ModifiableSetting, setting_name.upper()) + settings_to_modify = [ + config_file.SettingModification( + setting=modifiable_setting, + value=value_to_set, + ) + ] + with open(output_path, "r") as f: + expected_output = f.read() + shutil.copyfile(input_path, worker_config_path) + + # WHEN + config_file.ConfigFile.modify_config_file_settings( + config_path=worker_config_path, + settings_to_modify=settings_to_modify, + backup=False, + ) + + # THEN + written_config = worker_config_path.read_text() + assert expected_output == written_config diff --git a/test/integ/config/test_config_file.py b/test/integ/config/test_config_file.py new file mode 100644 index 00000000..eb123bfd --- /dev/null +++ b/test/integ/config/test_config_file.py @@ -0,0 +1,323 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +from __future__ import annotations +import glob +import shutil +from pathlib import Path + +import pytest + + +from deadline_worker_agent.config import config_file + +from .conftest import ( + TEST_CASE_DATA_BASE_DIR, + INPUT_CONFIG_FILENAME, +) + + +TEST_CASE_DATA_BASE_DIR = Path(__file__).parent / "data" +INPUT_CONFIG_FILENAME = "input.toml" +EXPECTED_OUTPUT_CONFIG_FILENAME = "output.toml" + + +class TestConfigFileModifySettings: + class TestMissingFromInput: + """Tests that when an there is no existing setting or commented-out setting in the TOML file + and a new value is provided, that the setting is added to the file along with TOML + comments that document the setting. All existing content in the TOML document including + comments and blank space should be preserved. + + This relies on conventional directory structure relative to the parent directory of this + test module: + + data/ + unset/ + / + input.toml + output.toml + + Where is the lower-case match of each enum value in + deadline_worker_agent.config.config_file.ModifiableSetting. + + The input.toml file serves as the existing input config file. + + The output.toml is the expected output file to be generated. The expectation is that the + setting denoted by in output.toml contains the output value to be applied + to input.toml. This value is obtained from the output file using tomllib/tomli and then used + to modify the input TOML document. + """ + + INPUT_FILES = glob.glob( + str(TEST_CASE_DATA_BASE_DIR / "missing" / "*" / INPUT_CONFIG_FILENAME) + ) + + @pytest.fixture(params=INPUT_FILES, ids=[Path(f).parts[-2] for f in INPUT_FILES]) + def input_path(self, request: pytest.FixtureRequest) -> str: + return request.param + + def test( + self, + input_path: str, + output_path: str, + setting_name: str, + value_to_set: str | bool, + worker_config_path: Path, + ) -> None: + # GIVEN + modifiable_setting = getattr(config_file.ModifiableSetting, setting_name.upper()) + settings_to_modify = [ + config_file.SettingModification( + setting=modifiable_setting, + value=value_to_set, + ) + ] + with open(output_path, "r") as f: + expected_output = f.read() + shutil.copyfile(input_path, worker_config_path) + + # WHEN + config_file.ConfigFile.modify_config_file_settings( + config_path=worker_config_path, + settings_to_modify=settings_to_modify, + backup=False, + ) + + # THEN + written_config = worker_config_path.read_text() + assert expected_output == written_config + + def test_coverage( + self, + ) -> None: + """Tests that we have coverage for all of the possible ModifiableSetting values""" + # GIVEN + attributes_covered = {Path(f).parts[-2].upper() for f in self.INPUT_FILES} + attributes = set(config_file.ModifiableSetting._member_names_) + + # THEN + assert attributes_covered == attributes + + class TestModifyExistingSettingInInput: + """Tests that when an there is an existing active setting in the TOML file and a new value, + is provided, that the setting is updated and all TOML comments and blank space are + preserved. + + This relies on conventional directory structure relative to the parent directory of this + test module: + + data/ + unset/ + / + input.toml + output.toml + + Where is the lower-case match of each enum value in + deadline_worker_agent.config.config_file.ModifiableSetting. + + The input.toml file serves as the existing input config file. + + The output.toml is the expected output file to be generated. The expectation is that the + setting denoted by in output.toml has a modified value from input.toml. This + value is obtained from the output file using tomllib/tomli and then used to modify the + input TOML document. + """ + + INPUT_FILES = glob.glob( + str(TEST_CASE_DATA_BASE_DIR / "existing" / "*" / INPUT_CONFIG_FILENAME) + ) + + @pytest.fixture(params=INPUT_FILES, ids=[Path(f).parts[-2] for f in INPUT_FILES]) + def input_path(self, request: pytest.FixtureRequest) -> str: + return request.param + + def test( + self, + input_path: str, + output_path: str, + setting_name: str, + value_to_set: str | bool, + worker_config_path: Path, + ) -> None: + # GIVEN + modifiable_setting = getattr(config_file.ModifiableSetting, setting_name.upper()) + settings_to_modify = [ + config_file.SettingModification( + setting=modifiable_setting, + value=value_to_set, + ) + ] + with open(output_path, "r") as f: + expected_output = f.read() + shutil.copyfile(input_path, worker_config_path) + + # WHEN + config_file.ConfigFile.modify_config_file_settings( + config_path=worker_config_path, + settings_to_modify=settings_to_modify, + backup=False, + ) + + # THEN + written_config = worker_config_path.read_text() + assert expected_output == written_config + + def test_coverage( + self, + ) -> None: + """Tests that we have coverage for all of the possible ModifiableSetting values""" + # GIVEN + attributes_covered = {Path(f).parts[-2].upper() for f in self.INPUT_FILES} + attributes = set(config_file.ModifiableSetting._member_names_) + + # THEN + assert attributes_covered == attributes + + class TestSettingValueUncommentsCommentedOutSetting: + """Tests that when an there is an existing setting in the TOML file that is commented out + and a new value is provided, that the commented-out setting becomes un-commented and its + value is updated. + + This relies on conventional directory structure relative to the parent directory of this + test module: + + data/ + unset/ + / + input.toml + output.toml + + Where is the lower-case match of each enum value in + deadline_worker_agent.config.config_file.ModifiableSetting. + + The input.toml file serves as the existing input config file. + + The output.toml is the expected output file to be generated. The expectation is that the + setting denoted by in output.toml has a modified value for the corresponding + commented-out setting in input.toml. This value is obtained from the output file using + tomllib/tomli and then used to modify the input TOML document. + """ + + INPUT_FILES = glob.glob( + str(TEST_CASE_DATA_BASE_DIR / "commented" / "*" / INPUT_CONFIG_FILENAME) + ) + + @pytest.fixture(params=INPUT_FILES, ids=[Path(f).parts[-2] for f in INPUT_FILES]) + def input_path(self, request: pytest.FixtureRequest) -> str: + return request.param + + def test( + self, + input_path: str, + output_path: str, + setting_name: str, + value_to_set: str | bool, + worker_config_path: Path, + ) -> None: + """The functional test. See class docstring""" + # GIVEN + modifiable_setting = getattr(config_file.ModifiableSetting, setting_name.upper()) + settings_to_modify = [ + config_file.SettingModification( + setting=modifiable_setting, + value=value_to_set, + ) + ] + with open(output_path, "r") as f: + expected_output = f.read() + shutil.copyfile(input_path, worker_config_path) + + # WHEN + config_file.ConfigFile.modify_config_file_settings( + config_path=worker_config_path, + settings_to_modify=settings_to_modify, + backup=False, + ) + + # THEN + written_config = worker_config_path.read_text() + assert expected_output == written_config + + def test_coverage( + self, + ) -> None: + """Tests that we have coverage for all of the possible ModifiableSetting values""" + # GIVEN + attributes_covered = {Path(f).parts[-2].upper() for f in self.INPUT_FILES} + attributes = set(config_file.ModifiableSetting._member_names_) + + # THEN + assert attributes_covered == attributes + + class TestUnsetCommentsOutInputSetting: + """Tests that when an there is an existing active setting in the TOML file that is being + unset, that the setting becomes commented out and all TOML comments and blank space are + preserved. + + This relies on conventional directory structure relative to the parent directory of this + test module: + + data/ + unset/ + / + input.toml + output.toml + + Where is the lower-case match of each enum value in + deadline_worker_agent.config.config_file.ModifiableSetting. + + The input.toml file serves as the existing input config file. The output.toml is the + expected output file to be generated with the setting denoted by commented + out. + """ + + INPUT_FILES = glob.glob( + str(TEST_CASE_DATA_BASE_DIR / "unset" / "*" / INPUT_CONFIG_FILENAME) + ) + + @pytest.fixture(params=INPUT_FILES, ids=[Path(f).parts[-2] for f in INPUT_FILES]) + def input_path(self, request: pytest.FixtureRequest) -> str: + return request.param + + def test( + self, + input_path: str, + output_path: str, + setting_name: str, + worker_config_path: Path, + ) -> None: + """The functional test. See class docstring""" + # GIVEN + value_to_set = None + modifiable_setting = getattr(config_file.ModifiableSetting, setting_name.upper()) + settings_to_modify = [ + config_file.SettingModification( + setting=modifiable_setting, + value=value_to_set, + ) + ] + with open(output_path, "r") as f: + expected_output = f.read() + shutil.copyfile(input_path, worker_config_path) + + # WHEN + config_file.ConfigFile.modify_config_file_settings( + config_path=worker_config_path, + settings_to_modify=settings_to_modify, + backup=False, + ) + + # THEN + written_config = worker_config_path.read_text() + assert expected_output == written_config + + def test_coverage( + self, + ) -> None: + """Tests that we have coverage for all of the possible ModifiableSetting values""" + # GIVEN + attributes_covered = {Path(f).parts[-2].upper() for f in self.INPUT_FILES} + attributes = set(config_file.ModifiableSetting._member_names_) + + # THEN + assert attributes_covered == attributes diff --git a/test/integ/conftest.py b/test/integ/conftest.py new file mode 100644 index 00000000..56224656 --- /dev/null +++ b/test/integ/conftest.py @@ -0,0 +1,11 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +import sys + +# Ignore platform-specific tests on other platforms +# https://docs.pytest.org/en/stable/example/pythoncollection.html#customizing-test-collection +collect_ignore: list[str] = [] +if sys.platform != "win32": + collect_ignore.append("windows") +elif sys.platform != "linux": + collect_ignore.append("linux") diff --git a/test/integ/windows/__init__.py b/test/integ/windows/__init__.py new file mode 100644 index 00000000..8d929cc8 --- /dev/null +++ b/test/integ/windows/__init__.py @@ -0,0 +1 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. diff --git a/test/integ/installer/test_windows_installer.py b/test/integ/windows/test_installer.py similarity index 99% rename from test/integ/installer/test_windows_installer.py rename to test/integ/windows/test_installer.py index 2abbec6f..26a44a2a 100644 --- a/test/integ/installer/test_windows_installer.py +++ b/test/integ/windows/test_installer.py @@ -39,9 +39,6 @@ WorkerAgentDirectories, ) -if sys.platform != "win32": - pytest.skip("Windows-specific tests", allow_module_level=True) - def test_user_existence(): current_user = win32api.GetUserNameEx(win32api.NameSamCompatible) diff --git a/test/unit/install/test_install.py b/test/unit/install/test_install.py index 71f498de..fc890045 100644 --- a/test/unit/install/test_install.py +++ b/test/unit/install/test_install.py @@ -6,6 +6,7 @@ from subprocess import CalledProcessError from typing import Generator from unittest.mock import MagicMock, patch +import sys import sysconfig import typing @@ -64,6 +65,8 @@ def expected_cmd( parsed_args.user, "--scripts-path", sysconfig.get_path("scripts"), + "--python-interpreter-path", + sys.executable, "--vfs-install-path", parsed_args.vfs_install_path, ] diff --git a/test/unit/windows/test_win_logon.py b/test/unit/windows/test_win_logon.py index e4186ced..2528d47e 100644 --- a/test/unit/windows/test_win_logon.py +++ b/test/unit/windows/test_win_logon.py @@ -43,7 +43,7 @@ def windows_session_user(self) -> Generator[MagicMock, None, None]: @fixture(autouse=True) def outside_session_0(self) -> Generator[MagicMock, None, None]: - with patch.object(win_logon_mod.win_service, "is_windows_session_zero") as mock: + with patch.object(win_logon_mod, "is_windows_session_zero") as mock: mock.return_value = False yield mock @@ -109,7 +109,7 @@ def chandle(self) -> Generator[MagicMock, None, None]: @fixture(autouse=True) def is_session_0(self) -> Generator[MagicMock, None, None]: - with patch.object(win_logon_mod.win_service, "is_windows_session_zero") as mock: + with patch.object(win_logon_mod, "is_windows_session_zero") as mock: mock.return_value = True yield mock diff --git a/test/unit/windows/test_win_service.py b/test/unit/windows/test_win_service.py index ddd0a88d..b228c84a 100644 --- a/test/unit/windows/test_win_service.py +++ b/test/unit/windows/test_win_service.py @@ -1,7 +1,5 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -from unittest.mock import patch - import pytest import sys @@ -11,69 +9,6 @@ from win32serviceutil import ServiceFramework from deadline_worker_agent.windows.win_service import WorkerAgentWindowsService -from deadline_worker_agent.windows import win_service - - -def test_get_current_process_session() -> None: - """Tests that the _get_current_process_session() function uses the expected pywin32 API calls""" - - # GIVEN - with ( - patch.object( - win_service.win32process, "GetCurrentProcessId" - ) as mock_get_current_process_id, - patch.object(win_service.win32ts, "ProcessIdToSessionId") as mock_process_id_to_session_id, - ): - # WHEN - result = win_service._get_current_process_session() - - # THEN - mock_get_current_process_id.assert_called_once_with() - mock_process_id_to_session_id.assert_called_once_with(mock_get_current_process_id.return_value) - assert result == mock_process_id_to_session_id.return_value - - -@pytest.mark.parametrize( - argnames="session_id,expected_result", - argvalues=( - pytest.param(0, True, id="session-zero"), - pytest.param(1, False, id="session-non-zero"), - ), -) -def test_is_windows_session_zero(session_id: int, expected_result: bool) -> None: - """Tests that the is_windows_session_zero() function returns true iff the return value of - _get_current_process_session is 0""" - - # GIVEN - # clear the cache decorator to ensure the function result is not cached between tests - win_service.is_windows_session_zero.cache_clear() - with patch.object(win_service, "_get_current_process_session", return_value=session_id): - # WHEN - result = win_service.is_windows_session_zero() - - # THEN - assert result == expected_result - - -def test_is_windows_session_zero_cached() -> None: - """Tests that the is_windows_session_zero() function caches the result between calls""" - - # GIVEN - # clear the cache decorator to ensure the function result is not cached on first run - win_service.is_windows_session_zero.cache_clear() - with patch.object( - win_service, "_get_current_process_session" - ) as mock_get_current_process_session: - # We make our mocked _get_current_process_session return different session IDs between calls - mock_get_current_process_session.side_effect = [0, 1] - first_result = win_service.is_windows_session_zero() - # WHEN - second_result = win_service.is_windows_session_zero() - - # THEN - assert first_result is True - assert second_result == first_result - mock_get_current_process_session.assert_called_once_with() def test_svc_name() -> None: diff --git a/test/unit/windows/test_win_session.py b/test/unit/windows/test_win_session.py new file mode 100644 index 00000000..365d08d6 --- /dev/null +++ b/test/unit/windows/test_win_session.py @@ -0,0 +1,73 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +from unittest.mock import patch + +import pytest +import sys + +if sys.platform != "win32": + pytest.skip("Windows-specific tests", allow_module_level=True) + +from deadline_worker_agent.windows import win_session + + +def test_get_current_process_session() -> None: + """Tests that the _get_current_process_session() function uses the expected pywin32 API calls""" + + # GIVEN + with ( + patch.object( + win_session.win32process, "GetCurrentProcessId" + ) as mock_get_current_process_id, + patch.object(win_session.win32ts, "ProcessIdToSessionId") as mock_process_id_to_session_id, + ): + # WHEN + result = win_session._get_current_process_session() + + # THEN + mock_get_current_process_id.assert_called_once_with() + mock_process_id_to_session_id.assert_called_once_with(mock_get_current_process_id.return_value) + assert result == mock_process_id_to_session_id.return_value + + +@pytest.mark.parametrize( + argnames="session_id,expected_result", + argvalues=( + pytest.param(0, True, id="session-zero"), + pytest.param(1, False, id="session-non-zero"), + ), +) +def test_is_windows_session_zero(session_id: int, expected_result: bool) -> None: + """Tests that the is_windows_session_zero() function returns true iff the return value of + _get_current_process_session is 0""" + + # GIVEN + # clear the cache decorator to ensure the function result is not cached between tests + win_session.is_windows_session_zero.cache_clear() + with patch.object(win_session, "_get_current_process_session", return_value=session_id): + # WHEN + result = win_session.is_windows_session_zero() + + # THEN + assert result == expected_result + + +def test_is_windows_session_zero_cached() -> None: + """Tests that the is_windows_session_zero() function caches the result between calls""" + + # GIVEN + # clear the cache decorator to ensure the function result is not cached on first run + win_session.is_windows_session_zero.cache_clear() + with patch.object( + win_session, "_get_current_process_session" + ) as mock_get_current_process_session: + # We make our mocked _get_current_process_session return different session IDs between calls + mock_get_current_process_session.side_effect = [0, 1] + first_result = win_session.is_windows_session_zero() + # WHEN + second_result = win_session.is_windows_session_zero() + + # THEN + assert first_result is True + assert second_result == first_result + mock_get_current_process_session.assert_called_once_with()