feat!: Create a script to build adaptor packaging artifacts (#87)

The breaking change is due to updating the adaptor runtime dependency to
0.4

- Also deprecate the binary name MayaAdaptor, and add the name
  maya-openjd
- Update the adaptor runtime dependency to 0.4, and fix up socket
  path into server path
- Update the submitter to provide both Rez and Conda packages metadata.
  The Rez package names are the same for now to keep backwards
  compatibility.
- Reduce code coverage requirement from 45 to 44, CI on one of the
  Python versions was a smidgeon below.

Signed-off-by: Mark Wiebe <[email protected]>

pyproject.toml

@@ -11,10 +11,12 @@ requires-python = ">=3.7"
 
 dependencies = [
     "deadline == 0.37.*",
-    "openjd-adaptor-runtime == 0.3.*",
+    "openjd-adaptor-runtime == 0.4.*",
 ]
 
 [project.scripts]
+maya-openjd = "deadline.maya_adaptor.MayaAdaptor:main"
+# The binary name 'MayaAdaptor' is deprecated.
 MayaAdaptor = "deadline.maya_adaptor.MayaAdaptor:main"
 
 [tool.hatch.build]
@@ -129,7 +131,7 @@ source = [
 
 [tool.coverage.report]
 show_missing = true
-fail_under = 45
+fail_under = 44
 
 [tool.semantic_release]
 # Can be removed or set to true once we are v1

scripts/create_adaptor_packaging_artifact.sh

@@ -0,0 +1,190 @@
+#!/usr/bin/env bash
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+set -xeou pipefail
+
+APP=maya
+ADAPTOR_NAME=deadline-cloud-for-$APP
+
+# This script generates an tar.gz artifact from $ADAPTOR_NAME and its dependencies
+# that can be used to create a package for running the adaptor.
+
+SCRIPTDIR=$(realpath $(dirname $0))
+
+SOURCE=0
+# Python 3.11 is for https://vfxplatform.com/ CY2024
+PYTHON_VERSION=3.11
+CONDA_PLATFORM=linux-64
+TAR_BASE=
+
+while [ $# -gt 0 ]; do
+    case "${1}" in
+    --source) SOURCE=1 ; shift ;;
+    --platform) CONDA_PLATFORM="$2" ; shift 2 ;;
+    --python) PYTHON_VERSION="$2" ; shift 2 ;;
+    --tar-base) TAR_BASE="$2" ; shift 2 ;;
+    *) echo "Unexpected option: $1"; exit 1 ;;
+  esac
+done
+
+if [ "$CONDA_PLATFORM" = "linux-64" ]; then
+    PYPI_PLATFORM=manylinux2014_x86_64
+elif [ "$CONDA_PLATFORM" = "win-64" ]; then
+    PYPI_PLATFORM=win_amd64
+elif [ "$CONDA_PLATFORM" = "osx-64" ]; then
+    PYPI_PLATFORM=macosx_10_9_x86_64
+else
+    echo "Unknown Conda operating system option --platform $CONDA_PLATFORM"
+    exit 1
+fi
+
+if [ "$TAR_BASE" = "" ]; then
+    TAR_BASE=$SCRIPTDIR/../$APP-openjd-py$PYTHON_VERSION-$CONDA_PLATFORM
+fi
+
+# Create a temporary prefix
+WORKDIR=$(mktemp -d adaptor-pkg.XXXXXXXXXX)
+function cleanup_workdir {
+    echo "Cleaning up $WORKDIR"
+    rm -rf $WORKDIR
+}
+trap cleanup_workdir EXIT
+
+PREFIX=$WORKDIR/prefix
+
+if [ "$CONDA_PLATFORM" = "win-64" ]; then
+    BINDIR=$PREFIX/Library/bin
+    PACKAGEDIR=$PREFIX/Library/opt/$ADAPTOR_NAME
+else
+    BINDIR=$PREFIX/bin
+    PACKAGEDIR=$PREFIX/opt/$ADAPTOR_NAME
+fi
+
+
+mkdir -p $PREFIX
+mkdir -p $PACKAGEDIR
+mkdir -p $BINDIR
+
+# Install the adaptor into the virtual env
+if [ $SOURCE = 1 ]; then
+    # In source mode, openjd-adaptor-runtime-for-python must be alongside this adaptor source
+    RUNTIME_INSTALLABLE=$SCRIPTDIR/../../openjd-adaptor-runtime-for-python
+    ADAPTOR_INSTALLABLE=$SCRIPTDIR/..
+
+    if [ "$CONDA_PLATFORM" = "win-64" ]; then
+        DEPS="pyyaml jsonschema pywin32"
+    else
+        DEPS="pyyaml jsonschema"
+    fi
+
+    for DEP in $DEPS; do
+        pip install \
+            --target $PACKAGEDIR \
+            --platform $PYPI_PLATFORM \
+            --python-version $PYTHON_VERSION \
+            --ignore-installed \
+            --only-binary=:all: \
+            $DEP
+    done
+
+    pip install \
+        --target $PACKAGEDIR \
+        --platform $PYPI_PLATFORM \
+        --python-version $PYTHON_VERSION \
+        --ignore-installed \
+        --no-deps \
+        $RUNTIME_INSTALLABLE
+    pip install \
+        --target $PACKAGEDIR \
+        --platform $PYPI_PLATFORM \
+        --python-version $PYTHON_VERSION \
+        --ignore-installed \
+        --no-deps \
+        $ADAPTOR_INSTALLABLE
+else
+    # In PyPI mode, PyPI and/or a CodeArtifact must have these packages
+    RUNTIME_INSTALLABLE=openjd-adaptor-runtime-for-python
+    ADAPTOR_INSTALLABLE=$ADAPTOR_NAME
+
+    pip install \
+        --target $PACKAGEDIR \
+        --platform $PYPI_PLATFORM \
+        --python-version $PYTHON_VERSION \
+        --ignore-installed \
+        --only-binary=:all: \
+        $RUNTIME_INSTALLABLE
+    pip install \
+        --target $PACKAGEDIR \
+        --platform $PYPI_PLATFORM \
+        --python-version $PYTHON_VERSION \
+        --ignore-installed \
+        --no-deps \
+        $ADAPTOR_INSTALLABLE
+fi
+
+
+# Remove the submitter code
+rm -r $PACKAGEDIR/deadline/*_submitter
+
+# Remove the bin dir if there is one
+if [ -d $PACKAGEDIR/bin ]; then
+    rm -r $PACKAGEDIR/bin
+fi
+
+PYSCRIPT="from pathlib import Path
+import sys
+reentry_exe = Path(sys.argv[0]).absolute()
+sys.path.append(str(reentry_exe.parent.parent / \"opt\" / \"$ADAPTOR_NAME\"))
+from deadline.${APP}_adaptor.${APP^}Adaptor.__main__ import main
+sys.exit(main(reentry_exe=reentry_exe))
+"
+
+cat <<EOF > $BINDIR/$APP-openjd
+#!/usr/bin/env python3.11
+$PYSCRIPT
+EOF
+
+# Temporary
+cp $BINDIR/$APP-openjd $BINDIR/${APP^}Adaptor
+
+chmod u+x $BINDIR/$APP-openjd $BINDIR/${APP^}Adaptor
+
+if [ $CONDA_PLATFORM = "win-64" ]; then
+    # Install setuptools to get cli-64.exe
+    mkdir -p $WORKDIR/tmp
+    pip install \
+        --target $WORKDIR/tmp \
+        --platform $PYPI_PLATFORM \
+        --python-version $PYTHON_VERSION \
+        --ignore-installed \
+        --no-deps \
+        setuptools
+
+    # Use setuptools' cli-64.exe to define the entry point
+    cat <<EOF > $BINDIR/$APP-openjd-script.py
+#!C:\\Path\\To\\Python.exe
+$PYSCRIPT
+EOF
+    cp $WORKDIR/tmp/setuptools/cli-64.exe $BINDIR/$APP-openjd.exe
+fi
+
+# Everything between the first "-" and the next "+" is the package version number
+PACKAGEVER=$(cd $PACKAGEDIR; echo deadline_cloud_for*)
+PACKAGEVER=${PACKAGEVER#*-}
+PACKAGEVER=${PACKAGEVER%+*}
+echo "Package version number is $PACKAGEVER"
+
+# Create the tar artifact
+GIT_TIMESTAMP="$(env TZ=UTC git log -1 --date=iso-strict-local --format="%ad")"
+pushd $PREFIX
+# See https://reproducible-builds.org/docs/archives/ for information about
+# these options
+#tar --mtime=$GIT_TIMESTAMP \
+#     --sort=name \
+#     --pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime \
+#    --owner=0 --group=0 --numeric-owner \
+#    -cf $TAR_BASE .
+# TODO Switch to the above command once the build environment has tar version > 1.28
+tar --owner=0 --group=0 --numeric-owner \
+    -cf $TAR_BASE-$PACKAGEVER.tar.gz .
+sha256sum $TAR_BASE-$PACKAGEVER.tar.gz
+popd

src/deadline/maya_adaptor/MayaAdaptor/__main__.py

@@ -11,21 +11,22 @@
 _logger = logging.getLogger(__name__)
 
 
-def main():
+def main(reentry_exe=None):
     _logger.info("About to start the MayaAdaptor")
 
     package_name = vars(sys.modules[__name__])["__package__"]
     if not package_name:
         raise RuntimeError(f"Must be run as a module. Do not run {__file__} directly")
 
     try:
-        EntryPoint(MayaAdaptor).start()
+        EntryPoint(MayaAdaptor).start(reentry_exe=reentry_exe)
     except Exception as e:
         _logger.error(f"Entrypoint failed: {e}")
-        sys.exit(1)
+        return 1
 
     _logger.info("Done MayaAdaptor main")
+    return 0
 
 
 if __name__ == "__main__":
-    main()
+    sys.exit(main())

src/deadline/maya_adaptor/MayaAdaptor/adaptor.py

@@ -139,11 +139,11 @@ def _wait_for_socket(self) -> str:
             str: The socket path the adaptor server is running on.
         """
         is_not_timed_out = self._get_timer(self._SERVER_START_TIMEOUT_SECONDS)
-        while (self._server is None or self._server.socket_path is None) and is_not_timed_out():
+        while (self._server is None or self._server.server_path is None) and is_not_timed_out():
             time.sleep(0.01)
 
-        if self._server is not None and self._server.socket_path is not None:
-            return self._server.socket_path
+        if self._server is not None and self._server.server_path is not None:
+            return self._server.server_path
 
         raise RuntimeError(
             "Could not find a socket path because the server did not finish initializing"
@@ -160,14 +160,14 @@ def _start_maya_server(self) -> None:
     def _start_maya_server_thread(self) -> None:
         """
         Starts the maya adaptor server in a thread.
-        Sets the environment variable "MAYA_ADAPTOR_SOCKET_PATH" to the socket the server is running
+        Sets the environment variable "MAYA_ADAPTOR_SERVER_PATH" to the socket the server is running
         on after the server has finished starting.
         """
         self._server_thread = threading.Thread(
             target=self._start_maya_server, name="MayaAdaptorServerThread"
         )
         self._server_thread.start()
-        os.environ["MAYA_ADAPTOR_SOCKET_PATH"] = self._wait_for_socket()
+        os.environ["MAYA_ADAPTOR_SERVER_PATH"] = self._wait_for_socket()
 
     def _get_regex_callbacks(self) -> list[RegexCallback]:
         """

src/deadline/maya_adaptor/MayaClient/maya_client.py

@@ -21,8 +21,8 @@
 
 
 class MayaClient(HTTPClientInterface):
-    def __init__(self, socket_path: str) -> None:
-        super().__init__(socket_path=socket_path)
+    def __init__(self, server_path: str) -> None:
+        super().__init__(server_path=server_path)
         self.actions.update(
             {
                 "renderer": self.set_renderer,
@@ -48,21 +48,21 @@ def graceful_shutdown(self, signum: int, frame: FrameType | None):
 
 
 def main():
-    socket_path = os.environ.get("MAYA_ADAPTOR_SOCKET_PATH")
-    if not socket_path:
+    server_path = os.environ.get("MAYA_ADAPTOR_SERVER_PATH")
+    if not server_path:
         raise OSError(
             "MayaClient cannot connect to the Adaptor because the environment variable "
-            "MAYA_ADAPTOR_SOCKET_PATH does not exist"
+            "MAYA_ADAPTOR_SERVER_PATH does not exist"
         )
 
-    if not os.path.exists(socket_path):
+    if not os.path.exists(server_path):
         raise OSError(
             "MayaClient cannot connect to the Adaptor because the socket at the path defined by "
-            "the environment variable MAYA_ADAPTOR_SOCKET_PATH does not exist. Got: "
-            f"{os.environ['MAYA_ADAPTOR_SOCKET_PATH']}"
+            "the environment variable MAYA_ADAPTOR_SERVER_PATH does not exist. Got: "
+            f"{os.environ['MAYA_ADAPTOR_SERVER_PATH']}"
         )
 
-    client = MayaClient(socket_path)
+    client = MayaClient(server_path)
     client.poll()
 
 

src/deadline/maya_submitter/maya_render_submitter.py

@@ -35,6 +35,7 @@
     LayerSelection,
 )
 from .cameras import get_renderable_camera_names, ALL_CAMERAS
+from ._version import version_tuple as adaptor_version_tuple
 from .ui.components.scene_settings_tab import SceneSettingsWidget
 from deadline.client.job_bundle.submission import AssetReferences
 
@@ -412,21 +413,27 @@ def _get_parameter_values(
             + f"{', '.join(parameter_overlap)}"
         )
 
-    # If we're overriding the adaptor with wheels, remove deadline_cloud_for_maya from the RezPackages
+    # If we're overriding the adaptor with wheels, remove the adaptor from the Packages parameters
     if settings.include_adaptor_wheels:
         rez_param = {}
-        # Find the RezPackages parameter definition
+        conda_param = {}
+        # Find the Packages parameter definition
         for param in queue_parameters:
             if param["name"] == "RezPackages":
                 rez_param = param
-                break
-        # Remove the deadline_cloud_for_maya rez package
+            if param["name"] == "CondaPackages":
+                conda_param = param
+        # Remove the deadline_cloud_for_maya/maya-openjd package
         if rez_param:
             rez_param["value"] = " ".join(
                 pkg
                 for pkg in rez_param["value"].split()
                 if not pkg.startswith("deadline_cloud_for_maya")
             )
+        if conda_param:
+            conda_param["value"] = " ".join(
+                pkg for pkg in conda_param["value"].split() if not pkg.startswith("maya-openjd")
+            )
 
     parameter_values.extend(
         {"name": param["name"], "value": param["value"]} for param in queue_parameters
@@ -510,6 +517,8 @@ def show_maya_render_submitter(parent, f=Qt.WindowFlags()) -> "Optional[SubmitJo
         all_layer_selectable_cameras
     )
 
+    all_renderers: set[str] = {layer_data.renderer_name for layer_data in render_layers}
+
     def on_create_job_bundle_callback(
         widget: SubmitJobToDeadlineDialog,
         job_bundle_dir: str,
@@ -634,10 +643,24 @@ def on_create_job_bundle_callback(
         output_directories=set(render_settings.output_directories),
     )
 
+    maya_version = maya.cmds.about(version=True)
+    adaptor_version = ".".join(str(v) for v in adaptor_version_tuple[:2])
+
+    # Need Maya and the Maya OpenJD application interface adaptor
+    rez_packages = f"mayaIO-{maya_version} deadline_cloud_for_maya"
+    conda_packages = f"maya={maya_version}.* maya-openjd={adaptor_version}.*"
+    # Add any additional renderers that are used
+    if "arnold" in all_renderers:
+        rez_packages += " mtoa"
+        conda_packages += " maya-mtoa"
+
     submitter_dialog = SubmitJobToDeadlineDialog(
         job_setup_widget_type=SceneSettingsWidget,
         initial_job_settings=render_settings,
-        initial_shared_parameter_values={"RezPackages": "mayaIO mtoa deadline_cloud_for_maya"},
+        initial_shared_parameter_values={
+            "RezPackages": rez_packages,
+            "CondaPackages": conda_packages,
+        },
         auto_detected_attachments=auto_detected_attachments,
         attachments=attachments,
         on_create_job_bundle_callback=on_create_job_bundle_callback,