feat!: Create a script to build adaptor package artifacts (#66)

The breaking change is due to updating the adaptor runtime dependency to
0.4

- Also deprecate the binary name HoudiniAdaptor, and add the name
  houdini-openjd
- Update the adaptor runtime dependency to 0.4, and fix up socket
  path into server path
- Update the submitter to provide both Rez and Conda packages metadata.
  The Rez package names are the same for now to keep backwards
  compatibility.
- Change the ROP menu from "Digital Assets" to "Farm" matching other
  render farm ROPs
- Adjusted the frame range logic to directly use the hou.Node
  properties

Signed-off-by: Mark Wiebe <[email protected]>
Signed-off-by: Morgan Epp <[email protected]>

pyproject.toml

@@ -11,10 +11,12 @@ requires-python = ">=3.9"
 
 dependencies = [
     "deadline == 0.37.*",
-    "openjd-adaptor-runtime == 0.3.*",
+    "openjd-adaptor-runtime == 0.4.*",
 ]
 
 [project.scripts]
+houdini-openjd = "deadline.houdini_adaptor.HoudiniAdaptor:main"
+# The binary name 'HoudiniAdaptor' is deprecated.
 HoudiniAdaptor = "deadline.houdini_adaptor.HoudiniAdaptor:main"
 
 [tool.hatch.build]
@@ -38,6 +40,7 @@ path = "hatch_custom_hook.py"
 destinations = [
   "src/deadline/houdini_adaptor",
   "src/deadline/houdini_submitter",
+  "src/deadline/houdini_submitter/python/deadline_cloud_for_houdini",
 ]
 
 [tool.hatch.build.targets.sdist]

scripts/create_adaptor_packaging_artifact.sh

@@ -0,0 +1,190 @@
+#!/usr/bin/env bash
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+set -xeou pipefail
+
+APP=houdini
+ADAPTOR_NAME=deadline-cloud-for-$APP
+
+# This script generates an tar.gz artifact from $ADAPTOR_NAME and its dependencies
+# that can be used to create a package for running the adaptor.
+
+SCRIPTDIR=$(realpath $(dirname $0))
+
+SOURCE=0
+# Python 3.11 is for https://vfxplatform.com/ CY2024
+PYTHON_VERSION=3.11
+CONDA_PLATFORM=linux-64
+TAR_BASE=
+
+while [ $# -gt 0 ]; do
+    case "${1}" in
+    --source) SOURCE=1 ; shift ;;
+    --platform) CONDA_PLATFORM="$2" ; shift 2 ;;
+    --python) PYTHON_VERSION="$2" ; shift 2 ;;
+    --tar-base) TAR_BASE="$2" ; shift 2 ;;
+    *) echo "Unexpected option: $1"; exit 1 ;;
+  esac
+done
+
+if [ "$CONDA_PLATFORM" = "linux-64" ]; then
+    PYPI_PLATFORM=manylinux2014_x86_64
+elif [ "$CONDA_PLATFORM" = "win-64" ]; then
+    PYPI_PLATFORM=win_amd64
+elif [ "$CONDA_PLATFORM" = "osx-64" ]; then
+    PYPI_PLATFORM=macosx_10_9_x86_64
+else
+    echo "Unknown Conda operating system option --platform $CONDA_PLATFORM"
+    exit 1
+fi
+
+if [ "$TAR_BASE" = "" ]; then
+    TAR_BASE=$SCRIPTDIR/../$APP-openjd-py$PYTHON_VERSION-$CONDA_PLATFORM
+fi
+
+# Create a temporary prefix
+WORKDIR=$(mktemp -d adaptor-pkg.XXXXXXXXXX)
+function cleanup_workdir {
+    echo "Cleaning up $WORKDIR"
+    rm -rf $WORKDIR
+}
+trap cleanup_workdir EXIT
+
+PREFIX=$WORKDIR/prefix
+
+if [ "$CONDA_PLATFORM" = "win-64" ]; then
+    BINDIR=$PREFIX/Library/bin
+    PACKAGEDIR=$PREFIX/Library/opt/$ADAPTOR_NAME
+else
+    BINDIR=$PREFIX/bin
+    PACKAGEDIR=$PREFIX/opt/$ADAPTOR_NAME
+fi
+
+
+mkdir -p $PREFIX
+mkdir -p $PACKAGEDIR
+mkdir -p $BINDIR
+
+# Install the adaptor into the virtual env
+if [ $SOURCE = 1 ]; then
+    # In source mode, openjd-adaptor-runtime-for-python must be alongside this adaptor source
+    RUNTIME_INSTALLABLE=$SCRIPTDIR/../../openjd-adaptor-runtime-for-python
+    ADAPTOR_INSTALLABLE=$SCRIPTDIR/..
+
+    if [ "$CONDA_PLATFORM" = "win-64" ]; then
+        DEPS="pyyaml jsonschema pywin32"
+    else
+        DEPS="pyyaml jsonschema"
+    fi
+
+    for DEP in $DEPS; do
+        pip install \
+            --target $PACKAGEDIR \
+            --platform $PYPI_PLATFORM \
+            --python-version $PYTHON_VERSION \
+            --ignore-installed \
+            --only-binary=:all: \
+            $DEP
+    done
+
+    pip install \
+        --target $PACKAGEDIR \
+        --platform $PYPI_PLATFORM \
+        --python-version $PYTHON_VERSION \
+        --ignore-installed \
+        --no-deps \
+        $RUNTIME_INSTALLABLE
+    pip install \
+        --target $PACKAGEDIR \
+        --platform $PYPI_PLATFORM \
+        --python-version $PYTHON_VERSION \
+        --ignore-installed \
+        --no-deps \
+        $ADAPTOR_INSTALLABLE
+else
+    # In PyPI mode, PyPI and/or a CodeArtifact must have these packages
+    RUNTIME_INSTALLABLE=openjd-adaptor-runtime-for-python
+    ADAPTOR_INSTALLABLE=$ADAPTOR_NAME
+
+    pip install \
+        --target $PACKAGEDIR \
+        --platform $PYPI_PLATFORM \
+        --python-version $PYTHON_VERSION \
+        --ignore-installed \
+        --only-binary=:all: \
+        $RUNTIME_INSTALLABLE
+    pip install \
+        --target $PACKAGEDIR \
+        --platform $PYPI_PLATFORM \
+        --python-version $PYTHON_VERSION \
+        --ignore-installed \
+        --no-deps \
+        $ADAPTOR_INSTALLABLE
+fi
+
+
+# Remove the submitter code
+rm -r $PACKAGEDIR/deadline/*_submitter
+
+# Remove the bin dir if there is one
+if [ -d $PACKAGEDIR/bin ]; then
+    rm -r $PACKAGEDIR/bin
+fi
+
+PYSCRIPT="from pathlib import Path
+import sys
+reentry_exe = Path(sys.argv[0]).absolute()
+sys.path.append(str(reentry_exe.parent.parent / \"opt\" / \"$ADAPTOR_NAME\"))
+from deadline.${APP}_adaptor.${APP^}Adaptor.__main__ import main
+sys.exit(main(reentry_exe=reentry_exe))
+"
+
+cat <<EOF > $BINDIR/$APP-openjd
+#!/usr/bin/env python3.11
+$PYSCRIPT
+EOF
+
+# Temporary
+cp $BINDIR/$APP-openjd $BINDIR/${APP^}Adaptor
+
+chmod u+x $BINDIR/$APP-openjd $BINDIR/${APP^}Adaptor
+
+if [ $CONDA_PLATFORM = "win-64" ]; then
+    # Install setuptools to get cli-64.exe
+    mkdir -p $WORKDIR/tmp
+    pip install \
+        --target $WORKDIR/tmp \
+        --platform $PYPI_PLATFORM \
+        --python-version $PYTHON_VERSION \
+        --ignore-installed \
+        --no-deps \
+        setuptools
+
+    # Use setuptools' cli-64.exe to define the entry point
+    cat <<EOF > $BINDIR/$APP-openjd-script.py
+#!C:\\Path\\To\\Python.exe
+$PYSCRIPT
+EOF
+    cp $WORKDIR/tmp/setuptools/cli-64.exe $BINDIR/$APP-openjd.exe
+fi
+
+# Everything between the first "-" and the next "+" is the package version number
+PACKAGEVER=$(cd $PACKAGEDIR; echo deadline_cloud_for*)
+PACKAGEVER=${PACKAGEVER#*-}
+PACKAGEVER=${PACKAGEVER%+*}
+echo "Package version number is $PACKAGEVER"
+
+# Create the tar artifact
+GIT_TIMESTAMP="$(env TZ=UTC git log -1 --date=iso-strict-local --format="%ad")"
+pushd $PREFIX
+# See https://reproducible-builds.org/docs/archives/ for information about
+# these options
+#tar --mtime=$GIT_TIMESTAMP \
+#     --sort=name \
+#     --pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime \
+#    --owner=0 --group=0 --numeric-owner \
+#    -cf $TAR_BASE .
+# TODO Switch to the above command once the build environment has tar version > 1.28
+tar --owner=0 --group=0 --numeric-owner \
+    -cf $TAR_BASE-$PACKAGEVER.tar.gz .
+sha256sum $TAR_BASE-$PACKAGEVER.tar.gz
+popd

scripts/install_dev_submitter.py

@@ -30,7 +30,7 @@ class HoudiniVersion:
         "19.5": "3.9",
     }
 
-    def __init__(self, arg_version: Optional[str]):
+    def __init__(self, arg_version: Optional[str] = None):
         version = self._get_houdini_version(arg_version)
         match = self.VERSION_REGEX.match(version)
         if match is None:
@@ -137,6 +137,7 @@ def install_submitter_package(houdini_version_arg: Optional[str], local_deps: li
     major_minor = houdini_version.major_minor()
 
     plugin_env_path = get_git_root() / "plugin_env"
+    os.makedirs(plugin_env_path, exist_ok=True)
     _build_deps_env(
         plugin_env_path,
         houdini_version.python_major_minor(),

src/deadline/houdini_adaptor/HoudiniAdaptor/adaptor.py

@@ -148,11 +148,11 @@ def _wait_for_socket(self) -> str:
             str: The socket path the adaptor server is running on.
         """
         is_timed_out = self._get_timer(self._SERVER_START_TIMEOUT_SECONDS)
-        while (self._server is None or self._server.socket_path is None) and not is_timed_out():
+        while (self._server is None or self._server.server_path is None) and not is_timed_out():
             time.sleep(0.01)
 
-        if self._server is not None and self._server.socket_path is not None:
-            return self._server.socket_path
+        if self._server is not None and self._server.server_path is not None:
+            return self._server.server_path
 
         raise RuntimeError("Could not find a socket because the server did not finish initializing")
 
@@ -167,15 +167,15 @@ def _start_houdini_server(self) -> None:
     def _start_houdini_server_thread(self) -> None:
         """
         Starts the houdini adaptor server in a thread.
-        Sets the environment variable "HOUDINI_ADAPTOR_SOCKET_PATH" to
+        Sets the environment variable "HOUDINI_ADAPTOR_SERVER_PATH" to
         the socket the server is running
         on after the server has finished starting.
         """
         self._server_thread = threading.Thread(
             target=self._start_houdini_server, name="HoudiniAdaptorServerThread"
         )
         self._server_thread.start()
-        os.environ["HOUDINI_ADAPTOR_SOCKET_PATH"] = self._wait_for_socket()
+        os.environ["HOUDINI_ADAPTOR_SERVER_PATH"] = self._wait_for_socket()
 
     @property
     def validators(self) -> AdaptorDataValidators:

src/deadline/houdini_adaptor/HoudiniClient/houdini_client.py

@@ -30,8 +30,8 @@ class HoudiniClient(HTTPClientInterface):
     Client that runs in Houdini for the Houdini Adaptor
     """
 
-    def __init__(self, socket_path: str) -> None:
-        super().__init__(socket_path=socket_path)
+    def __init__(self, server_path: str) -> None:
+        super().__init__(server_path=server_path)
         self.actions.update(HoudiniHandler().action_dict)
 
     def close(self, args: Optional[dict] = None) -> None:
@@ -42,21 +42,21 @@ def graceful_shutdown(self, signum: int, frame: FrameType | None):
 
 
 def main():
-    socket_path = os.environ.get("HOUDINI_ADAPTOR_SOCKET_PATH")
-    if not socket_path:
+    server_path = os.environ.get("HOUDINI_ADAPTOR_SERVER_PATH")
+    if not server_path:
         raise OSError(
             "HoudiniClient cannot connect to the Adaptor because the environment variable "
-            "HOUDINI_ADAPTOR_SOCKET_PATH does not exist"
+            "HOUDINI_ADAPTOR_SERVER_PATH does not exist"
         )
 
-    if not os.path.exists(socket_path):
+    if not os.path.exists(server_path):
         raise OSError(
             "HoudiniClient cannot connect to the Adaptor because the socket at the path defined by "
-            "the environment variable HOUDINI_ADAPTOR_SOCKET_PATH does not exist. Got: "
-            f"{os.environ['HOUDINI_ADAPTOR_SOCKET_PATH']}"
+            "the environment variable HOUDINI_ADAPTOR_SERVER_PATH does not exist. Got: "
+            f"{os.environ['HOUDINI_ADAPTOR_SERVER_PATH']}"
         )
 
-    client = HoudiniClient(socket_path)
+    client = HoudiniClient(server_path)
     client.poll()
 
 

src/deadline/houdini_submitter/otls/deadline_cloud.hda/Driver_1deadline__cloud/Tools.shelf

@@ -12,7 +12,7 @@
     <toolMenuContext name="network">
       <contextOpType>$HDA_TABLE_AND_NAME</contextOpType>
     </toolMenuContext>
-    <toolSubmenu>Digital Assets</toolSubmenu>
+    <toolSubmenu>Farm</toolSubmenu>
     <script scriptType="python"><![CDATA[import drivertoolutils
 
 drivertoolutils.genericTool(kwargs, '$HDA_NAME')]]></script>

src/deadline/houdini_submitter/python/deadline_cloud_for_houdini/queue_parameters.py

@@ -5,15 +5,11 @@
 
 import hou
 
+from ._version import version_tuple as adaptor_version_tuple
 from deadline.client.api._queue_parameters import get_queue_parameter_definitions
 from deadline.client.job_bundle.parameters import JobParameter
 
 
-_QUEUE_ENVIRONMENT_SPECIAL_DEFAULTS = {
-    "RezPackages": "houdini deadline_cloud_for_houdini",
-}
-
-
 def _get_queue_parameter_groups(
     queue_parameter_definitions: list[JobParameter],
 ) -> tuple[dict[str, list[JobParameter]], list[JobParameter]]:
@@ -141,8 +137,13 @@ def _get_equivalent_bool(original_value: str) -> Optional[bool]:
 
 
 def _get_default_value(param: JobParameter) -> tuple[Union[str, int, float], ...]:
-    if param["name"] in _QUEUE_ENVIRONMENT_SPECIAL_DEFAULTS:
-        return (_QUEUE_ENVIRONMENT_SPECIAL_DEFAULTS[param["name"]],)
+    houdini_version = ".".join(hou.applicationVersionString().split(".")[:2])
+    adaptor_version = ".".join(str(v) for v in adaptor_version_tuple[:2])
+
+    if param["name"] == "RezPackages":
+        return (f"houdini-{houdini_version}.* deadline_cloud_for_houdini",)
+    elif param["name"] == "CondaPackages":
+        return (f"houdini={houdini_version}.* houdini-openjd={adaptor_version}.*",)
     elif "default" in param:
         return (param["default"],)
     else:

src/deadline/houdini_submitter/python/deadline_cloud_for_houdini/submitter.py

@@ -8,9 +8,6 @@
 from pathlib import Path
 
 from deadline.client.job_bundle._yaml import deadline_yaml_dump
-from deadline.client.job_bundle.adaptors import (
-    parse_frame_range,
-)
 from deadline.client import api
 from deadline.client.job_bundle.submission import AssetReferences
 from deadline.client.job_bundle import create_job_history_bundle_dir
@@ -148,6 +145,7 @@ def _get_parameter_values(node: hou.Node) -> dict[str, Any]:
             {"name": "deadline:targetTaskRunStatus", "value": initial_status},
             {"name": "deadline:maxFailedTasksCount", "value": failed_tasks_limit},
             {"name": "deadline:maxRetriesPerTask", "value": task_retry_limit},
+            {"name": "HipFile", "value": _get_hip_file()},
             *get_queue_parameter_values_as_openjd(node),
         ]
     }
@@ -200,13 +198,11 @@ def _get_job_template(rop: hou.Node) -> dict[str, Any]:
         task_data_contents.append("frame: {{Task.Param.Frame}}\n")
         task_data_contents.append("ignore_input_nodes: true\n")
         # step
-        frame_list = "{start}-{stop}:{step}".format(**n)
+        frame_range = "{start}-{stop}:{step}".format(**n)
         step = {
             "name": n["name"],
             "parameterSpace": {
-                "taskParameterDefinitions": [
-                    {"name": "Frame", "range": parse_frame_range(frame_list), "type": "INT"}
-                ]
+                "taskParameterDefinitions": [{"name": "Frame", "range": frame_range, "type": "INT"}]
             },
             "stepEnvironments": environments,
             "script": {