diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index dd351d8..f6028d2 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -1,12 +1,31 @@ name: "\U0001F41B Bug Report" description: Report a bug -title: "Bug: TITLE" -labels: ["bug"] +title: "Bug: (short bug description)" +labels: ["bug", "needs triage"] body: + - type: markdown + attributes: + value: | + Thank you for taking the time to fill out this bug report! + + ⚠️ If the bug that you are reporting is a security-related issue or security vulnerability, + then please do not create a report via this template. Instead please + notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/) + or directly via email to [AWS Security](aws-security@amazon.com). + + - type: textarea + id: description + attributes: + label: Describe the bug + description: What is the problem? A clear and concise description of the bug. + validations: + required: true + - type: textarea id: expected_behaviour attributes: label: Expected Behaviour + description: What did you expect to happen? validations: required: true @@ -14,6 +33,7 @@ body: id: current_behaviour attributes: label: Current Behaviour + description: What actually happened? Please include as much detail as you can. validations: required: true @@ -21,13 +41,24 @@ body: id: reproduction_steps attributes: label: Reproduction Steps + description: | + Please provide as much detail as you can to help us understand how we can reproduce the bug. + Step by step instructions and self-contained code snippets are ideal. validations: required: true - type: textarea - id: code_snippet + id: environment attributes: - label: Code Snippet + label: Environment + description: Please provide information on the environment and software versions that you are using to reproduce the bug. + value: | + At minimum: + 1. Operating system: (e.g. Windows Server 2022; Amazon Linux 2023; etc.) + 2. Version of Maya: + 3: Version of this package: + 4. If this is from a version installed by the Deadline Cloud Submitter installer, then what version of the submitter installer? + + Please share other details about your environment that you think might be relevant to reproducing the bug. validations: required: true - diff --git a/.github/ISSUE_TEMPLATE/doc.yml b/.github/ISSUE_TEMPLATE/doc.yml index 883623b..7a95fad 100644 --- a/.github/ISSUE_TEMPLATE/doc.yml +++ b/.github/ISSUE_TEMPLATE/doc.yml @@ -1,8 +1,8 @@ name: "📕 Documentation Issue" description: Issue in the documentation -title: "Docs: TITLE" -labels: ["documenation"] +title: "Docs: (short description of the issue)" +labels: ["documenation", "needs triage"] body: - type: textarea id: documentation_issue diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index ed7f957..d284468 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -1,17 +1,37 @@ name: "\U0001F680 Feature Request" description: Request a new feature -title: "Feature request: TITLE" -labels: ["feature"] +title: "Feature request: (short description of the feature)" +labels: ["feature", "needs triage"] body: - type: textarea - id: use_case + id: problem attributes: - label: Use Case + label: Describe the problem + description: | + Help us understand the problem that you are trying to solve, and why it is important to you. + Provide as much detail as you are able. validations: required: true + - type: textarea id: proposed_solution attributes: label: Proposed Solution + description: | + Describe your proposed feature that you see solving this problem for you. If you have a + full or partial prototype implementation then please open a draft pull request and link to + it here as well. + validations: + required: true + + - type: textarea + id: use_case + attributes: + label: Example Use Cases + description: | + Provide some sample code snippets or shell scripts that show how **you** would use this feature as + you have proposed it. validations: required: true + + diff --git a/.github/ISSUE_TEMPLATE/maintenance.yml b/.github/ISSUE_TEMPLATE/maintenance.yml index a0f98ff..f2440eb 100644 --- a/.github/ISSUE_TEMPLATE/maintenance.yml +++ b/.github/ISSUE_TEMPLATE/maintenance.yml @@ -1,17 +1,19 @@ name: "🛠️ Maintenance" description: Some type of improvement -title: "Maintenance: TITLE" -labels: ["feature"] +title: "Maintenance: (short description of the issue)" +labels: ["feature", "needs triage"] body: - type: textarea id: description attributes: label: Description + description: Describe the improvement and why it is important to do. validations: required: true - type: textarea id: solution attributes: label: Solution + description: Provide any ideas you have for how the suggestion can be implemented. validations: required: true diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index ff5619c..93625d8 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,3 +1,5 @@ +Fixes: ** + ### What was the problem/requirement? (What/Why) ### What was the solution? (How) @@ -6,16 +8,40 @@ ### How was this change tested? +See [DEVELOPMENT.md](https://github.com/aws-deadline/deadline-cloud-for-maya/blob/mainline/DEVELOPMENT.md) for information on running tests. + +- Have you run the unit tests? + ### Did you run the "Job Bundle Output Tests"? If not, why not? If so, paste the test results here. +See the "Integration Tests" subsection of the +[Running Submitter Tests](https://github.com/aws-deadline/deadline-cloud-for-maya/blob/mainline/DEVELOPMENT.md#running-submitter-tests) +section of DEVELOPMENT.md for information on these tests. + ``` Required: paste the contents of job_bundle_output_tests/test-job-bundle-results.txt here ``` ### Was this change documented? +- Did you update all relevant docstrings for Python functions that you modified? +- Should the README.md, DEVELOPMENT.md, or other documents in the repository's docs/ directory be updated along with your change? +- Should the schema files for the adaptor's init-data or run-data be updated? + ### Is this a breaking change? +A breaking change is one that modifies a public contract in some way or otherwise changes functionality of this application in a way +that is not backwards compatible. Examples of changes that are breaking include: + +1. Adding a new required value to the init-data or run-data of the adaptor; +2. Deleting or renaming a value in the init-data or run-data of the adaptor; and +3. Otherwise modifying the interface of the adaptor such that a job submitted with an older version of the Maya submitter plug-in + will not work a version of the adaptor that includes your modification. + +If so, then please describe the changes that users of this package must make to update their scripts, or Python applications. Also, +please ensure that the title of your commit follows our conventional commit guidelines in +[CONTRIBUTING.md](https://github.com/aws-deadline/deadline-cloud-for-maya/blob/mainline/CONTRIBUTING.md#conventional-commits) for breaking changes. + ---- *By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.* diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 454d82f..21651de 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,62 +6,106 @@ documentation, we greatly value feedback and contributions from our community. Please read through this document before submitting any issues or pull requests to ensure we have all the necessary information to effectively respond to your bug report or contribution. +Table of contents: + +* [Reporting Bugs/Feature Requests](#reporting-bugsfeature-requests) +* [Development](#development) + * [Finding contributions to work on](#finding-contributions-to-work-on) + * [Talk with us first](#talk-with-us-first) + * [Contributing via Pull Requests](#contributing-via-pull-requests) + * [Conventional Commits](#conventional-commits) +* [Licensing](#licensing) + ## Reporting Bugs/Feature Requests We welcome you to use the GitHub issue tracker to report bugs or suggest features. When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already -reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: - -* A reproducible test case or series of steps -* The version of our code being used -* Any modifications you've made relevant to the bug -* Anything unusual about your environment or deployment +reported the issue. Please try to include as much information as you can. ## Development -Please see [DEVELOPMENT.md](./DEVELOPMENT.md) for more information. +We welcome you to contribute features and bug fixes via a [pull request](https://help.github.com/articles/creating-a-pull-request/). +If you are new to contributing to GitHub repositories, then you may find the +[GitHub documentation on collaborating with the fork and pull model](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/getting-started/about-collaborative-development-models#fork-and-pull-model) +informative; this is the model that we follow. + +Please see [DEVELOPMENT.md](./DEVELOPMENT.md) for information about how to navigate this package's +code base and development practices. + +### Finding contributions to work on + +If you are not sure what you would like to contribute, then looking at the existing issues is a great way to find +something to contribute on. Looking at +[issues that have the "help wanted" or "good first issue" labels](https://github.com/aws-deadline/deadline-cloud-for-maya/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22%2C%22help+wanted%22) +are a good place to start, but please dive into any issue that interests you whether it has those labels or not. -## Contributing via Pull Requests +### Talk with us first + +We ask that you please [open a feature request issue](https://github.com/aws-deadline/deadline-cloud-for-maya/issues/new/choose) +(if one does not already exist) and talk with us before posting a pull request that contains a significant amount of work, +or one that proposes a change to a public interface such as to the interface of a publicly exported Python function or to +the command-line interfaces' commands or arguments. We want to make sure that your time and effort is respected by working +with you to design the change before you spend much of your time on it. If you want to create a draft pull request to show what +you are thinking and then talk with us, then that works with us as well. + +We prefer that this package contain primarily features that are useful to many users of it, rather than features that are specific +to niche workflows. If you have a feature in mind, but are not sure whether it is niche or not then please +[open a feature request issue](https://github.com/aws-deadline/deadline-cloud-for-maya/issues/new/choose). We will do our best to help +you make that assessment, and posting a public issue will help others find your feature idea and add their support if they +would also find it useful. + +### Contributing via Pull Requests Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: -1. You are working against the latest source on the *main* branch. +1. You are working against the latest source on the *mainline* branch. 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. +4. Your pull request will be focused on a single change - it is easier for us to understand when a change is focused rather + than changing multiple things at once. To send us a pull request, please: 1. Fork the repository. -2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. -3. Ensure local tests pass. -4. Commit to your fork using clear commit messages. +2. Modify the source and add tests for your change; please focus on the specific change you are contributing. + If you also reformat all the code, it will be hard for us to focus on your change. + Please see [DEVELOPMENT.md](./DEVELOPMENT.md) for tips. +3. Ensure tests pass. Please see the [Testing](./DEVELOPMENT.md#testing) section for information on tests. +4. Commit to your fork using clear commit messages. Note that all AWS Deadline Cloud GitHub repositories require the use + of [conventional commit](#conventional-commits) syntax for the title of your commit. 5. Send us a pull request, answering any default questions in the pull request interface. 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. GitHub provides additional documentation on [forking a repository](https://help.github.com/articles/fork-a-repo/) and [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). -## Finding contributions to work on - -Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. - +### Conventional commits -## Code of Conduct +The commits in this repository are all required to use [conventional commit syntax](https://www.conventionalcommits.org/en/v1.0.0/) +in their title to help us identify the kind of change that is being made, automatically generate the changelog, and +automatically identify next release version number. Only the first commit that deviates from mainline in your pull request +must adhere to this requirement. -This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). -For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact -opensource-codeofconduct@amazon.com with any additional questions or comments. +We ask that you use these commit types in your commit titles: +* `feat` - When the pull request adds a new feature or functionality; +* `fix` - When the pull request is implementing a fix to a bug; +* `test` - When the pull request is only implementing an addition or change to tests or the testing infrastructure; +* `docs` - When the pull request is primarily implementing an addition or change to the package's documentation; +* `refactor` - When the pull request is implementing only a refactor of existing code; +* `ci` - When the pull request is implementing a change to the CI infrastructure of the packge; +* `chore` - When the pull request is a generic maintenance task. -## Security issue notifications +We also require that the type in your conventional commit title end in an exclaimation point (e.g. `feat!` or `fix!`) +if the pull request should be considered to be a breaking change in some way. Please also include a "BREAKING CHANGE" footer +in the description of your commit in this case ([example](https://www.conventionalcommits.org/en/v1.0.0/#commit-message-with-both--and-breaking-change-footer)). +Examples of breaking changes include any that implements a backwards-imcompatible change to a public Python interface, +the command-line interface, or the like. -We take all security reports seriously. When we receive such reports, we will -investigate and subsequently address any potential vulnerabilities as quickly -as possible. If you discover a potential security issue in this project, please -notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/) -or directly via email to [AWS Security](aws-security@amazon.com). Please do not -create a public GitHub issue in this project. +If you need change a commit message, then please see the +[GitHub documentation on the topic](https://docs.github.com/en/pull-requests/committing-changes-to-your-project/creating-and-editing-commits/changing-a-commit-message) +to guide you. ## Licensing diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 5f24978..546dea2 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -1,79 +1,221 @@ # Development documentation -This package has two active branches: +This documentation provides guidance on developer workflows for working with the code in this repository. -- `mainline` -- For active development. This branch is not intended to be consumed by other packages. Any commit to this branch may break APIs, dependencies, and so on, and thus break any consumer without notice. -- `release` -- The official release of the package intended for consumers. Any breaking releases will be accompanied with an increase to this package's interface version. +Table of Contents: -## Build / Test / Release +* [Development Environment Setup](#development-environment-setup) +* [The Development Loop](#the-development-loop) + * [Submitter Development Workflow](#submitter-development-workflow) + * [Running the Plug-In](#running-the-plug-in) + * [Making Code Changes](#making-submitter-code-changes) + * [Running Tests](#running-submitter-tests) + * [Adaptor Development Workflow](#adaptor-development-workflow) + * [Running the Adaptor Locally](#running-the-adaptor-locally) + * [Running the Adaptor on a Farm](#running-the-adaptor-on-a-farm) + * [Testing the Adaptor](#testing-the-adaptor) -### Build the package +## Development Environment Setup -```bash -hatch build -``` +To develop the Python code in this repository you will need: -### Run tests +1. Python 3.9 or higher. We recommend [mise](https://github.com/jdx/mise) if you would like to run more than one version + of Python on the same system. When running unit tests against all supported Python versions, for instance. +2. The [hatch](https://github.com/pypa/hatch) package installed (`pip install --upgrade hatch`) into your Python environment. +3. An install of a supported version of Maya. +4. A valid AWS Account. +5. An AWS Deadline Cloud Farm to run jobs on. We recommend following the quickstart in the Deadline Cloud console to create a + Queue with the default Queue Environment, and a Service Managed Fleet. -```bash -hatch run test -``` +You can develop on a Linux, MacOS, or Windows workstation. -### Run linting +## Software Architecture -```bash -hatch run lint -``` +If you are not already familiar with the architecture of the Maya submitter plugin and adaptor application in this repository +then we suggest going over [] for an overview of how these components are constructed, and how they function. + +## The Development Loop + +We have configured [hatch](https://github.com/pypa/hatch) commands to support a standard development loop. You can run the following +from any directory of this repository: + +* `hatch build` - To build the installable Python wheel and sdist packages into the `dist/` directory. +* `hatch run test` - To run the PyTest unit tests found in the `test/unit` directory. See [Testing](#testing). +* `hatch run all:test` - To run the PyTest unit tests against all available supported versions of Python. +* `hatch run lint` - To check that the package's formatting adheres to our standards. +* `hatch run fmt` - To automatically reformat all code to adhere to our formatting standards. +* `hatch shell` - Enter a shell environment that will have Python set up to import your development version of this package. +* `hatch env prune` - Delete all of your isolated workspace [environments](https://hatch.pypa.io/1.12/environment/) + for this package. +* `hatch run install` - Install the DeadlineCloudForMaya plugin from this repository into a temporary directory within this repository. + +Note: Hatch uses [environments](https://hatch.pypa.io/1.12/environment/) to isolate the Python development workspace +for this package from your system or virtual environment Python. If your build/test run is not making sense, then +sometimes pruning (`hatch env prune`) all of these environments for the package can fix the issue. + +### Submitter Development Workflow + +The submitter plug-in generates job bundles to submit to AWS Deadline Cloud. Developing a change +to the submitter involves iteratively changing the plug-in code, then running the plug-in within Maya +to generate or submit a job bundle, inspecting the generated job bundle to ensure that it is as you expect, +and ultimately running that job to ensure that it works as desired. + +#### Running the Plug-In -### Run formatting +First, create a development installation of the submitter plug-in by running: ```bash -hatch run fmt +hatch run install ``` -### Run tests for all supported Python versions +This will create a `/plugin_env` directory in this repository with the submitter plug-in and all of its +dependencies installed into it. Then, activate the hatch shell environment, set the environment variable to enable +developer options in the plugin, and run maya: ```bash -hatch run all:test +hatch shell +export DEADLINE_ENABLE_DEVELOPER_OPTIONS=true +maya ``` -## Use development Submitter in Maya +You will need to load the plug-in within Maya once the application has completed loading. In the main menu bar, go to +Windows > Settings/Preferences > Plug-In Manager and you will find that `DeadlineCloudForMaya.py` is available as a +plug-in. Check the checkbox to have Maya load the plug-in and create the AWSDeadline tray for you. Click the icon on +the tray to open the submitter. + +You can use the "Export Bundle" option in the submitter to save the job bundle for a submission to your local disk +to inspect it, or the "Submit" button (after selecting your Deadline Cloud Farm and Queue in the submitter UI) to +submit the job to your farm to run. + +#### Making Submitter Code Changes + +Whenever you modify code for the plug-in, or one of its supporting Python libraries, you will need to re-run +the `hatch run install` to repackage the changes, and then uncheck then check the option to load the plug-in in +Maya's Plug-In Manager; if you do not reload the plug-in in Maya, then your changes will not take effect. + +#### Running Submitter Tests + +The tests for the plug-in have two forms: + +1. Unit tests - Small tests that are narrowly focused on ensuring that function-level behavior of the + implementation behaves as it is expected to. These can always be run locally on your workstation without + requiring an AWS account. +2. Integration tests - In-application tests that verify that job submissions generate expected job bundles. + +##### Unit Tests + +Unit tests are all located under the `test/deadline_submitter_for_maya/unit` directory of this repository. If you are adding +or modifying functionality, then you will almost always want to be writing one or more unit tests to demonstrate that your +logic behaves as expected and that future changes do not accidentally break your change. + +To run the unit tests, simply use hatch: ```bash -hatch run install -hatch shell +hatch test ``` -Then launch Maya from that terminal. -A development version of deadline-cloud-for-maya is then available to be loaded. +##### Integration Tests -## Submitter Development Workflow +Integration tests are built in to the plug-in itself. They are available as the `TEST` button in the AWSDeadline +shelf when you run Maya with the environment variable `DEADLINE_ENABLE_DEVELOPER_OPTIONS=true`. If you are adding +or modifying functionality that will affect the content of a generated job bundle then you will likely want to +be writing or modifying an integration test for your change. -WARNING: This workflow installs additional Python packages into your Maya's python distribution. +To run the integration tests: +1. Select the `TEST` button from the AWSDeadline shelf; and then +2. Select the `/job_bundle_output_tests` directory within this repository. -1. Create a development location within which to do your git checkouts. For example `~/deadline-clients`. Clone packages from this directory with commands like `git clone git@github.com:casillas2/deadline-cloud-for-maya.git`. You'll also want the `deadline-cloud` and `openjd` repos. -2. Switch to your Maya directory, like `cd "C:\Program Files\Autodesk\Maya2024"`. -3. Run `.\mayapy -m pip install -e C:\Users\\deadline-clients\deadline-cloud` to install the AWS Deadline Cloud Client Library in edit mode. -4. Run `.\mayapy -m pip install -e C:\Users\\deadline-clients\openjd-adaptor-runtime-for-python` to install the Open Job Description Adaptor Runtime Library in edit mode. -5. Run `.\mayapy -m pip install -e C:\Users\\deadline-clients\deadline-cloud-for-maya` to install the Maya submitter in edit mode. -6. Edit (create if missing) your `~/Documents/maya/2024/Maya.env` file, and add the following lines to it: +The test results will be saved to a file called `test-job-bundle-results.txt` within the test directory +that you selected. - ```bash - DEADLINE_ENABLE_DEVELOPER_OPTIONS=true - MAYA_MODULE_PATH=C:\Users\\deadline-clients\deadline-cloud-for-maya\maya_submitter_plugin - ``` +### Adaptor Development Workflow + +The maya adaptor is a command-line application (named `maya-openjd`) that interfaces with the Maya application. +You will need the `maya` executable available in your PATH for the adaptor to be able to run Maya. + +When developing a change to the Maya adaptor we recommend running the adaptor locally on your workstation, +and running and adding to the unit tests until you are comfortable that your change looks like it is working as you expect. +Testing locally like this will allow you to iterate faster on your change than the alternative of testing by +submitting jobs to Deadline Cloud to run using your modified adaptor. + +#### Running the Adaptor Locally + +To run the adaptor you will first need to create two files: + +1. An `init-data.yaml` (or `init-data.json`) file that contains the information passed to the adaptor + during its initialization phase. The schema for this file can be found at + `src/deadline/maya_adaptor/MayaAdaptor/schemas/init_data.schema.json`, and examples of init data can + be found in the `template.yaml` files spread throughout this repository. +2. A `run-data.yaml` (or `run-data.json`) file that contains the information passed to the adaptor + to do a single Task run. The schema for this file can be found at + `src/deadline/maya_adaptor/MayaAdaptor/schemas/run_data.schema.json`, and examples of init data can + be found in the `template.yaml` files spread throughout this repository. + +To run the adaptor once you have created an `init-data.yaml` and `run-data.yaml` file to test with: + +1. Ensure that `maya` can be run directly in your terminal by putting its location in your PATH environment variable. +2. Enter the hatch shell development environment by running `hatch shell` +3. Run the `maya-openjd` commmand-line with arguments that exercise your code change. + +The adaptor has two modes of operation: + +1. Running directly via the `maya-openjd run` subcommand; or +2. Running as a background daemon via subcommands of the `maya-openjd daemon` subcommand. + +We recommend primarily developing using the `maya-openjd run` subcommand as it is simpler to operate +for rapid development iterations, but that you should also ensure that your change works with the background +daemon mode before calling your change complete. + +The basic command to run the `maya-openjd` run command will look like: - The developer options add a shelf TEST button you can use to run the tests from the `job_bundle_output_tests` directory. -7. Run Maya. Go to Windows > Settings/Preferences > Plug-In Manager and you will find that `DeadlineCloudForMaya.py` is available as a plugin. Check the checkbox to have Maya load the plugin and create the Deadline tray for you. Click the icon on the tray to open the submitter. +```bash +maya-openjd run \ + --init-data file:// \ + --run-data file:// +``` + +The equivalent run with the `maya-openjd daemon` subcommand looks like: + +```bash +# The daemon start command requires that the connection-info file not already exist. +test -f connection-info.json || rm connection-info.json + +# This starts up a background process running the adaptor, and runs the `on_init` and `on_start` +# methods of the adaptor. +maya-openjd daemon start \ + --init-data file:// \ + --connection-file file://connection-info.json + +# This connects to the already running adaptor, via the information in the connection-info.json file, +# and runs the adaptor's `on_run` method. +maya-openjd daemon run \ + --run-data file:// \ + --connection-file file://connection-info.json + +# This connects to the already running adaptor to instruct it to shutdown the maya application +# and then exit. +maya-openjd daemon stop \ + --connection-file file://connection-info.json +``` + +#### Running the Adaptor on a Farm -## Application Interface Adaptor Development Workflow +If you have made modifications to the adaptor and wish to test your modifications on a live Deadline Cloud Farm +with real jobs, then we recommend using a [Service Managed Fleet](https://docs.aws.amazon.com/deadline-cloud/latest/userguide/smf-manage.html) +for your testing. We recommend performing this style of test if you have made any modifications that might interact with Deadline Cloud's +job attachments feature, or that could interact with path mapping in any way. We have implemented a developer feature in the Maya submitter +plug-in that submits the Python wheel files for your modified adaptor along with your job submission and uses the modified adaptor to +run the submitted job. -You can work on the adaptor alongside your submitter development workflow using a Deadline Cloud -farm that uses a service-managed fleet. You'll need to perform the following steps to substitute -your build of the adaptor for the one in the service. +You'll need to perform the following steps to substitute your build of the adaptor for the one in the service. -1. Use the development location from the Submitter Development Workflow. Make sure you're running Maya with `set DEADLINE_ENABLE_DEVELOPER_OPTIONS=true` enabled. -2. Build wheels for `openjd_adaptor_runtime`, `deadline` and `deadline_cloud_for_maya`, place them in a "wheels" folder in `deadline-cloud-for-maya`. A script is provided to do this, just execute from `deadline-cloud-for-maya`: +1. Using the submitter development workflow (See [Running the Plug-In](#running-the-plug-in)), make sure that you are + running Maya with `DEADLINE_ENABLE_DEVELOPER_OPTIONS=true` enabled. +2. Clone the [deadline-cloud](https://github.com/aws-deadline/deadline-cloud) and + [openjd-adaptor-runtime-for-python](https://github.com/OpenJobDescription/openjd-adaptor-runtime-for-python) repositories beside + this one, and ensure that you `git checkout release` in each to checkout the latest `release` branch. +2. Build wheels for `openjd_adaptor_runtime`, `deadline` and `deadline_cloud_for_maya`, place them in a "wheels" folder in `deadline-cloud-for-maya`. + A script is provided to do this, just execute from `deadline-cloud-for-maya`: ```bash # If you don't have the build package installed already @@ -92,3 +234,15 @@ your build of the adaptor for the one in the service. ``` 3. Open the Maya integrated submitter, and in the Job-Specific Settings tab, enable the option 'Include Adaptor Wheels'. This option is only visible when the environment variable `DEADLINE_ENABLE_DEVELOPER_OPTIONS` is set to `true`. Then submit your test job. + +#### Testing the Adaptor + +Unit tests are all located under the `test/deadline_submitter_for_maya/unit` directory of this repository. If you are adding +or modifying functionality, then you will almost always want to be writing one or more unit tests to demonstrate that your +logic behaves as expected and that future changes do not accidentally break your change. + +To run the unit tests, simply use hatch: + +```bash +hatch test +``` diff --git a/README.md b/README.md index d3ba056..716d368 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,19 @@ [![python](https://img.shields.io/pypi/pyversions/deadline-cloud-for-maya.svg?style=flat)](https://pypi.python.org/pypi/deadline-cloud-for-maya) [![license](https://img.shields.io/pypi/l/deadline-cloud-for-maya.svg?style=flat)](https://github.com/aws-deadline/deadline-cloud-for-maya/blob/mainline/LICENSE) -AWS Deadline Cloud for Maya is a python package that allows users to create [AWS Deadline Cloud][deadline-cloud] jobs from within Maya. Using the [Open Job Description (OpenJD) Adaptor Runtime][openjd-adaptor-runtime] this package also provides a command line application that adapts Maya's command line interface to support the [OpenJD specification][openjd]. +AWS Deadline Cloud for Maya is a Python package that supports creating and running Audodesk Maya jobs within [AWS Deadline Cloud](deadline-cloud). +It provides both the implementation of a Maya plug-in for your workstation that helps you offload the computation for your rendering workloads +to [AWS Deadline Cloud](deadline-cloud) to free up your workstation's compute for other tasks, and the implementation of a command-line +adaptor application based on the [Open Job Description (OpenJD) Adaptor Runtime][openjd-adaptor-runtime] that improves AWS Deadline Cloud's +ability to run Maya efficiently on your render farm. [deadline-cloud]: https://docs.aws.amazon.com/deadline-cloud/latest/userguide/what-is-deadline-cloud.html [deadline-cloud-client]: https://github.com/aws-deadline/deadline-cloud [openjd]: https://github.com/OpenJobDescription/openjd-specifications/wiki [openjd-adaptor-runtime]: https://github.com/OpenJobDescription/openjd-adaptor-runtime-for-python [openjd-adaptor-runtime-lifecycle]: https://github.com/OpenJobDescription/openjd-adaptor-runtime-for-python/blob/release/README.md#adaptor-lifecycle +[service-managed-fleets]: https://docs.aws.amazon.com/deadline-cloud/latest/userguide/smf-manage.html +[default-queue-environment]: https://docs.aws.amazon.com/deadline-cloud/latest/userguide/create-queue-environment.html#conda-queue-environment ## Compatibility @@ -20,46 +26,114 @@ This library requires: 1. Python 3.9 or higher; and 1. Linux, Windows, or a macOS operating system. -## Submitter +## Versioning + +This package's version follows [Semantic Versioning 2.0](https://semver.org/), but is still considered to be in its +initial development, thus backwards incompatible versions are denoted by minor version bumps. To help illustrate how +versions will increment during this initial development stage, they are described below: + +1. The MAJOR version is currently 0, indicating initial development. +2. The MINOR version is currently incremented when backwards incompatible changes are introduced to the public API. +3. The PATCH version is currently incremented when bug fixes or backwards compatible changes are introduced to the public API. + +## Getting Started + +This Maya integration for AWS Deadline Cloud has two components that you will need to install: + +1. The Maya submitter plug-in must be installed on the workstation that you will use to submit jobs; and +2. The Maya adaptor must be installed on all of your AWS Deadline Cloud worker hosts that will be running the Maya jobs that you submit. + +Before submitting any large, complex, or otherwise compute-heavy Maya render jobs to your farm using the submitter and adaptor that you +set up, we strongly recommend that you construct a simple test scene that can be rendered quickly and submit renders of that +scene to your farm to ensure that your setup is correctly functioning. -This package provides a Maya plugin that creates jobs for AWS Deadline Cloud using the [AWS Deadline Cloud client library][deadline-cloud-client]. Based on the loaded scene it determines the files required, allows the user to specify render options, and builds an [OpenJD template][openjd] that defines the workflow. +### Maya Submitter Plug-in -## Adaptor +The Maya submitter plug-in creates a shelf button in your Maya UI that can be used to submit jobs to AWS Deadline Cloud. Clicking this button +reveals a UI creates a job submission for AWS Deadline Cloud using the [AWS Deadline Cloud client library][deadline-cloud-client]. +It automatically determines the files required based on the loaded scene, allows the user to specify render options, builds an +[Open Job Description template][openjd] that defines the workflow, and submits the job to the farm and queue of your chosing. -The Maya Adaptor implements the [OpenJD][openjd-adaptor-runtime] interface that allows render workloads to launch Maya and feed it commands. This gives the following benefits: -* a standardized render application interface, -* sticky rendering, where the application stays open between tasks, -* path mapping, that enables cross-platform rendering +To install the submitter plug-in: -Jobs created by the submitter use this adaptor by default. +1. Install the `deadline-cloud-for-maya` Python package, with `mayapy`, into the Maya installation on your workstation by following + Maya's official online documentation (e.g. [For Maya 2024](maya-2024-mayapy)). +2. Copy the contents of the + [`/maya_submitter_plugin` directory](https://github.com/aws-deadline/deadline-cloud-for-maya/tree/release/maya_submitter_plugin) + from the release branch of [deadline-cloud-for-maya GitHub repository](https://github.com/aws-deadline/deadline-cloud-for-maya) into + a directory in Maya's module search paths. See the `MAYA_MODULE_PATH` section of Maya's official documentation + (e.g. [For Maya 2024](maya-2024-module-path)) for a list of the default search search paths on your system. +3. Within Maya, enable the DealineCloudForMaya plug-in in Maya's Plug-in Manager. + (Main menu bar: Windows -> Settings/Preferences -> Plug-in Manager) +4. To supply AWS account credentials for the submitter to use when submitting a job you can either: + 1. [Install and set up the Deadline Cloud Monitor](deadline-cloud-monitor-setup), and then log in to the monitor. Logging in + to the monitor will make AWS credentials available to the submitter, automatically. + 2. Set up an AWS credentials profile [as you would for the AWS CLI](aws-cli-credentials), and select that profile for the submitter + to use. + 3. Or default to your AWS EC2 instance profile credentials if you are running a workstation in the cloud. -### Getting Started +[maya-2024-mayapy]: https://help.autodesk.com/view/MAYAUL/2024/ENU/?guid=GUID-72A245EC-CDB4-46AB-BEE0-4BBBF9791627 +[maya-2024-module-path]: https://help.autodesk.com/view/MAYAUL/2024/ENU/?guid=GUID-228CCA33-4AFE-4380-8C3D-18D23F7EAC72 +[deadline-cloud-monitor-setup]: https://docs.aws.amazon.com/deadline-cloud/latest/userguide/submitter.html#install-deadline-cloud-monitor +[aws-cli-credentials]: https://docs.aws.amazon.com/cli/v1/userguide/cli-chap-authentication.html + +### Maya Adaptor + +Jobs created by this submitter require this adaptor be installed on your worker hosts, and that both the installed adaptor +and the Maya executable be available on the PATH of the user that will be running your jobs. + +The adaptor application is a command-line Python-based application that enhances the functionality of Maya for running +within a render farm like Deadline Cloud. It's primary purpose for existing is to add a "sticky rendering" functionality +where a single process instance of Maya is able to load the scene file and then dynamically be instructed to perform +desired renders without needing to close and re-launch Maya between them, though it also supplies additional benefit +such as support for path mapping, and reporting the progress of your render to Deadline Cloud. The alternative +to "sticky rendering" is that Maya would need to be run separately for each render that is done, and close afterwards. +Some scenes can take 10's of minutes just to load for rendering, so being able to keep the application open and loaded between +renders can be a significant time-saving optimization; particularly when the render itself is quick. + +If you are using the [default Queue Environment](default-queue-environment), or an equivalent, to run your jobs, then the adaptor will be +automatically made available to your job. Otherwise, you will need to install the adaptor. The adaptor can be installed by the standard python packaging mechanisms: ```sh $ pip install deadline-cloud-for-maya ``` -After installation it can then be used as a command line tool: +After installation it can then you can test that it has been installed properly by running the following as the same +user that runs your jobs: ```sh $ maya-openjd --help ``` For more information on the commands the OpenJD adaptor runtime provides, see [here][openjd-adaptor-runtime-lifecycle]. -## Versioning +### Maya Software Availability in AWS Deadline Cloud Service Managed Fleets -This package's version follows [Semantic Versioning 2.0](https://semver.org/), but is still considered to be in its -initial development, thus backwards incompatible versions are denoted by minor version bumps. To help illustrate how -versions will increment during this initial development stage, they are described below: +You will need to ensure that the version of Maya that you want to run is available on the worker host when you are using +AWS Deadline Cloud's [Service Managed Fleets](service-managed-fleets) to run jobs; +hosts do not have any rendering applications pre-installed. The standard way of accomplishing this is described +[in the service documentation](https://docs.aws.amazon.com/deadline-cloud/latest/developerguide/provide-applications.html). +You can find a list of the versions of Maya that are available by default +[in the user guide](https://docs.aws.amazon.com/deadline-cloud/latest/userguide/create-queue-environment.html#conda-queue-environment). -1. The MAJOR version is currently 0, indicating initial development. -2. The MINOR version is currently incremented when backwards incompatible changes are introduced to the public API. -3. The PATCH version is currently incremented when bug fixes or backwards compatible changes are introduced to the public API. +## Modifying Jobs Before Submitting + +To submit a job, the submitter first generates a [Job Bundle](job-bundle), and then uses functionality from the +[Deadline](deadline-cloud-client) package to submit the Job Bundle to your render farm to run. If you need to modify +the job that will be submitted to your farm, then you can use the "Export Bundle" button in the submitter to export the +Job Bundle to a location of your choice, modify that bundle, and then use the [Deadline Cloud application](deadline-cloud-client) +to submit that bundle to your farm. + +[job-bundle]: https://docs.aws.amazon.com/deadline-cloud/latest/developerguide/build-job-bundle.html ## Security -See [CONTRIBUTING](https://github.com/aws-deadline/deadline-cloud-for-maya/blob/release/CONTRIBUTING.md#security-issue-notifications) for more information. +We take all security reports seriously. When we receive such reports, we will +investigate and subsequently address any potential vulnerabilities as quickly +as possible. If you discover a potential security issue in this project, please +notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/) +or directly via email to [AWS Security](aws-security@amazon.com). Please do not +create a public GitHub issue in this project. ## Telemetry diff --git a/docs/software_arch.md b/docs/software_arch.md new file mode 100644 index 0000000..c210446 --- /dev/null +++ b/docs/software_arch.md @@ -0,0 +1,75 @@ +# Software Architecture + +This document provides an overview of the Maya submitter plug-in and adaptor that are in this repository. +The intent is to help you have a basic understanding of what the applications are doing to give you context +to understand what you are looking at when diving through the code. This is not a comprehensive deep dive of +the implementation. + +## Maya Submitter Plug-in + +The Maya plug-in is contructed in two parts: +1. A very bare-bones plug-in [module](https://help.autodesk.com/view/MAYAUL/2024/ENU/?guid=Maya_SDK_Distributing_Maya_Plug_ins_DistributingUsingModules_html) + in the `/maya_submitter_plugin` directory. +2. The `deadline.maya_submitter` Python package located in `/src/deadline/maya_submitter` that provides all of the actual business logic for the plugin. + +### Plug-in Module + +The plug-in module is mainly two files: + +1. `DeadlineCloudForMaya.mod` -- The [module description file](https://help.autodesk.com/view/MAYAUL/2024/ENU/?guid=Maya_SDK_Distributing_Maya_Plug_ins_DistributingUsingModules_ModuleDescriptionFiles_html) + for the plug-in. +2. `plug-ins/DeadlineCloudForMaya.py` -- The implementation of Maya's plug-in interface for the plugin. There is not much to this code. + It is just building the shelf in Maya for AWSDeadline, and creating the button(s) in that shelf for the users to click to launch the submitter. + Clicking the submitter button calls a function in the `deadline.maya_submitter` package to create the UI, and do all of the submitter functionality. + +### `deadline.maya_submitter` + +This Python module implements all of the functionality of the Maya plug-in itself. The entrypoints that Maya integrates with +are all defined in `deadline.maya_submitter.mel_commands`. These simply create a UI window to display to the user. + +Fundamentally, what this submitter is doing is creating a [Job Bundle](https://docs.aws.amazon.com/deadline-cloud/latest/developerguide/build-job-bundle.html) +and using the GUI creation code in the [`deadline` Python package](https://pypi.org/project/deadline/) to generate the UI +that is displayed to the user. The important parts to know about in a job bundle are: + +1. The job template file. The submitter code dynamically generates the template based on properties of the specific scene file + that is loaded. For example, it may contain a Step for each layer of the scene to render. + Note: All job template files are currently derived from a standard static job template located at + `src/deadline/maya_submitter/default_maya_job_template.yaml`. +2. Asset references. These are the files that the job, when submitted, will require to be able to run. The submitter contains code + that introspects the scene that is loaded to automatically discover these, and the submitter UI allows the end-user to modify + the list of files. + +The job submission itself is handled by functionality within the `deadline` package that is hooked up when the UI is created. + +## Maya Adaptor Application + +See the [README](../README.md#maya-adaptor) for background on what purpose the adaptor application serves. + +The implementation of the adaptor for Maya has two parts: + +1. The adaptor application itself whose code is located in `src/deadline/maya_adaptor/MayaAdaptor`. This is the + implementation of the command-line application (named `maya-openjd`) that is run by Jobs created by the Maya submitter. +2. A "MayaClient" application located in `src/deadline/maya_adaptor/MayaClient`. This is an application that is + run within Maya itself by the adaptor application when it launches Maya. The MayaClient facilitates communication + between the adaptor itself and the running Maya application; communication to tell Maya to, say, load a scene file, + or render frame 20 of the loaded scene. + +The adaptor application itself is built using the [Open Job Description Adaptor Runtime](https://github.com/OpenJobDescription/openjd-adaptor-runtime-for-python) +package. This package supplies the application entrypoint that defines and parses the command-line subcommands and options, as well as +the business logic that drives the state machine of the adaptor itself. Please see the README for the runtime package for information on +the lifecycle states of an adaptor, and the command line options that are available. + +Digging through the code for the adaptor, you will find that the `MayaAdaptor.on_start()` method is where the Maya application is started. +Rather than starting it with command-line options that say to render a specific scene, the application is started with arguments that +tell Maya to run the "MayaClient" application. This application is, essentially, a secure web server that is running over named pipes +rather than network sockets. The adaptor sends the client commands (look for calls to `enqueue_action()` in the adaptor) to instruct Maya +to do things, and then waits for the results of those actions to take effect. + +You can see the definitions of the available commands, and the actions that they take by inspecting `src/deadline/MayaClient/maya_client.py`. You'll +notice that the commands that it directly defines are minimal, and that the set of commands that are available is updated when the adaptor sends +it a command to set the rendererer being used. Each renderer has its own command handler defined under `src/deadline/MayaClient/render_handlers`. + +The final thing to be aware of is that the adaptor defines a number of stdout/stderr handlers. These are registered when launching the Maya process +via the `LoggingSubprocess` class. Each handler defines a regex that is compared against the output stream of Maya itself, and code that is run +when that regex is matched in Maya's output. This allows the adaptor to, say, translate the rendering progress status from Maya into a form +that can be understood and reported to Deadline Cloud. diff --git a/hatch.toml b/hatch.toml index b26ee67..44129a4 100644 --- a/hatch.toml +++ b/hatch.toml @@ -22,7 +22,7 @@ lint = [ install = "python {root}/scripts/install_dev_submitter.py {args:}" [[envs.all.matrix]] -python = ["3.8", "3.9", "3.10", "3.11"] +python = ["3.9", "3.10", "3.11", "3.12"] [envs.default.env-vars] SKIP_BOOTSTRAP_TEST_RESOURCES="True" diff --git a/pyproject.toml b/pyproject.toml index 8dfe355..604e2bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ classifiers = [ dependencies = [ "deadline == 0.48.*", - "openjd-adaptor-runtime >= 0.7,< 0.9", + "openjd-adaptor-runtime == 0.8.*", ] [project.urls] diff --git a/requirements-testing.txt b/requirements-testing.txt index 3ae0a78..63cee99 100644 --- a/requirements-testing.txt +++ b/requirements-testing.txt @@ -7,3 +7,4 @@ mypy == 1.* black == 24.* ruff == 0.4.* moto[s3,sts] == 5.* +pipgrip == 0.10.*