diff --git a/pyproject.toml b/pyproject.toml index 9d475c3..68fd477 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,10 +11,12 @@ requires-python = ">=3.9" dependencies = [ "deadline == 0.37.*", - "openjd-adaptor-runtime == 0.3.*", + "openjd-adaptor-runtime == 0.4.*", ] [project.scripts] +houdini-openjd = "deadline.houdini_adaptor.HoudiniAdaptor:main" +# The binary name 'HoudiniAdaptor' is deprecated. HoudiniAdaptor = "deadline.houdini_adaptor.HoudiniAdaptor:main" [tool.hatch.build] @@ -38,6 +40,7 @@ path = "hatch_custom_hook.py" destinations = [ "src/deadline/houdini_adaptor", "src/deadline/houdini_submitter", + "src/deadline/houdini_submitter/python/deadline_cloud_for_houdini", ] [tool.hatch.build.targets.sdist] diff --git a/scripts/create_adaptor_packaging_artifact.sh b/scripts/create_adaptor_packaging_artifact.sh new file mode 100755 index 0000000..0403420 --- /dev/null +++ b/scripts/create_adaptor_packaging_artifact.sh @@ -0,0 +1,190 @@ +#!/usr/bin/env bash +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +set -xeou pipefail + +APP=houdini +ADAPTOR_NAME=deadline-cloud-for-$APP + +# This script generates an tar.gz artifact from $ADAPTOR_NAME and its dependencies +# that can be used to create a package for running the adaptor. + +SCRIPTDIR=$(realpath $(dirname $0)) + +SOURCE=0 +# Python 3.11 is for https://vfxplatform.com/ CY2024 +PYTHON_VERSION=3.11 +CONDA_PLATFORM=linux-64 +TAR_BASE= + +while [ $# -gt 0 ]; do + case "${1}" in + --source) SOURCE=1 ; shift ;; + --platform) CONDA_PLATFORM="$2" ; shift 2 ;; + --python) PYTHON_VERSION="$2" ; shift 2 ;; + --tar-base) TAR_BASE="$2" ; shift 2 ;; + *) echo "Unexpected option: $1"; exit 1 ;; + esac +done + +if [ "$CONDA_PLATFORM" = "linux-64" ]; then + PYPI_PLATFORM=manylinux2014_x86_64 +elif [ "$CONDA_PLATFORM" = "win-64" ]; then + PYPI_PLATFORM=win_amd64 +elif [ "$CONDA_PLATFORM" = "osx-64" ]; then + PYPI_PLATFORM=macosx_10_9_x86_64 +else + echo "Unknown Conda operating system option --platform $CONDA_PLATFORM" + exit 1 +fi + +if [ "$TAR_BASE" = "" ]; then + TAR_BASE=$SCRIPTDIR/../$APP-openjd-py$PYTHON_VERSION-$CONDA_PLATFORM +fi + +# Create a temporary prefix +WORKDIR=$(mktemp -d adaptor-pkg.XXXXXXXXXX) +function cleanup_workdir { + echo "Cleaning up $WORKDIR" + rm -rf $WORKDIR +} +trap cleanup_workdir EXIT + +PREFIX=$WORKDIR/prefix + +if [ "$CONDA_PLATFORM" = "win-64" ]; then + BINDIR=$PREFIX/Library/bin + PACKAGEDIR=$PREFIX/Library/opt/$ADAPTOR_NAME +else + BINDIR=$PREFIX/bin + PACKAGEDIR=$PREFIX/opt/$ADAPTOR_NAME +fi + + +mkdir -p $PREFIX +mkdir -p $PACKAGEDIR +mkdir -p $BINDIR + +# Install the adaptor into the virtual env +if [ $SOURCE = 1 ]; then + # In source mode, openjd-adaptor-runtime-for-python must be alongside this adaptor source + RUNTIME_INSTALLABLE=$SCRIPTDIR/../../openjd-adaptor-runtime-for-python + ADAPTOR_INSTALLABLE=$SCRIPTDIR/.. + + if [ "$CONDA_PLATFORM" = "win-64" ]; then + DEPS="pyyaml jsonschema pywin32" + else + DEPS="pyyaml jsonschema" + fi + + for DEP in $DEPS; do + pip install \ + --target $PACKAGEDIR \ + --platform $PYPI_PLATFORM \ + --python-version $PYTHON_VERSION \ + --ignore-installed \ + --only-binary=:all: \ + $DEP + done + + pip install \ + --target $PACKAGEDIR \ + --platform $PYPI_PLATFORM \ + --python-version $PYTHON_VERSION \ + --ignore-installed \ + --no-deps \ + $RUNTIME_INSTALLABLE + pip install \ + --target $PACKAGEDIR \ + --platform $PYPI_PLATFORM \ + --python-version $PYTHON_VERSION \ + --ignore-installed \ + --no-deps \ + $ADAPTOR_INSTALLABLE +else + # In PyPI mode, PyPI and/or a CodeArtifact must have these packages + RUNTIME_INSTALLABLE=openjd-adaptor-runtime-for-python + ADAPTOR_INSTALLABLE=$ADAPTOR_NAME + + pip install \ + --target $PACKAGEDIR \ + --platform $PYPI_PLATFORM \ + --python-version $PYTHON_VERSION \ + --ignore-installed \ + --only-binary=:all: \ + $RUNTIME_INSTALLABLE + pip install \ + --target $PACKAGEDIR \ + --platform $PYPI_PLATFORM \ + --python-version $PYTHON_VERSION \ + --ignore-installed \ + --no-deps \ + $ADAPTOR_INSTALLABLE +fi + + +# Remove the submitter code +rm -r $PACKAGEDIR/deadline/*_submitter + +# Remove the bin dir if there is one +if [ -d $PACKAGEDIR/bin ]; then + rm -r $PACKAGEDIR/bin +fi + +PYSCRIPT="from pathlib import Path +import sys +reentry_exe = Path(sys.argv[0]).absolute() +sys.path.append(str(reentry_exe.parent.parent / \"opt\" / \"$ADAPTOR_NAME\")) +from deadline.${APP}_adaptor.${APP^}Adaptor.__main__ import main +sys.exit(main(reentry_exe=reentry_exe)) +" + +cat < $BINDIR/$APP-openjd +#!/usr/bin/env python3.11 +$PYSCRIPT +EOF + +# Temporary +cp $BINDIR/$APP-openjd $BINDIR/${APP^}Adaptor + +chmod u+x $BINDIR/$APP-openjd $BINDIR/${APP^}Adaptor + +if [ $CONDA_PLATFORM = "win-64" ]; then + # Install setuptools to get cli-64.exe + mkdir -p $WORKDIR/tmp + pip install \ + --target $WORKDIR/tmp \ + --platform $PYPI_PLATFORM \ + --python-version $PYTHON_VERSION \ + --ignore-installed \ + --no-deps \ + setuptools + + # Use setuptools' cli-64.exe to define the entry point + cat < $BINDIR/$APP-openjd-script.py +#!C:\\Path\\To\\Python.exe +$PYSCRIPT +EOF + cp $WORKDIR/tmp/setuptools/cli-64.exe $BINDIR/$APP-openjd.exe +fi + +# Everything between the first "-" and the next "+" is the package version number +PACKAGEVER=$(cd $PACKAGEDIR; echo deadline_cloud_for*) +PACKAGEVER=${PACKAGEVER#*-} +PACKAGEVER=${PACKAGEVER%+*} +echo "Package version number is $PACKAGEVER" + +# Create the tar artifact +GIT_TIMESTAMP="$(env TZ=UTC git log -1 --date=iso-strict-local --format="%ad")" +pushd $PREFIX +# See https://reproducible-builds.org/docs/archives/ for information about +# these options +#tar --mtime=$GIT_TIMESTAMP \ +# --sort=name \ +# --pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime \ +# --owner=0 --group=0 --numeric-owner \ +# -cf $TAR_BASE . +# TODO Switch to the above command once the build environment has tar version > 1.28 +tar --owner=0 --group=0 --numeric-owner \ + -cf $TAR_BASE-$PACKAGEVER.tar.gz . +sha256sum $TAR_BASE-$PACKAGEVER.tar.gz +popd diff --git a/scripts/install_dev_submitter.py b/scripts/install_dev_submitter.py index 104ad16..4c7136a 100644 --- a/scripts/install_dev_submitter.py +++ b/scripts/install_dev_submitter.py @@ -30,7 +30,7 @@ class HoudiniVersion: "19.5": "3.9", } - def __init__(self, arg_version: Optional[str]): + def __init__(self, arg_version: Optional[str] = None): version = self._get_houdini_version(arg_version) match = self.VERSION_REGEX.match(version) if match is None: @@ -137,6 +137,7 @@ def install_submitter_package(houdini_version_arg: Optional[str], local_deps: li major_minor = houdini_version.major_minor() plugin_env_path = get_git_root() / "plugin_env" + os.makedirs(plugin_env_path, exist_ok=True) _build_deps_env( plugin_env_path, houdini_version.python_major_minor(), diff --git a/src/deadline/houdini_adaptor/HoudiniAdaptor/adaptor.py b/src/deadline/houdini_adaptor/HoudiniAdaptor/adaptor.py index db62b1f..04e61f6 100644 --- a/src/deadline/houdini_adaptor/HoudiniAdaptor/adaptor.py +++ b/src/deadline/houdini_adaptor/HoudiniAdaptor/adaptor.py @@ -148,11 +148,11 @@ def _wait_for_socket(self) -> str: str: The socket path the adaptor server is running on. """ is_timed_out = self._get_timer(self._SERVER_START_TIMEOUT_SECONDS) - while (self._server is None or self._server.socket_path is None) and not is_timed_out(): + while (self._server is None or self._server.server_path is None) and not is_timed_out(): time.sleep(0.01) - if self._server is not None and self._server.socket_path is not None: - return self._server.socket_path + if self._server is not None and self._server.server_path is not None: + return self._server.server_path raise RuntimeError("Could not find a socket because the server did not finish initializing") @@ -167,7 +167,7 @@ def _start_houdini_server(self) -> None: def _start_houdini_server_thread(self) -> None: """ Starts the houdini adaptor server in a thread. - Sets the environment variable "HOUDINI_ADAPTOR_SOCKET_PATH" to + Sets the environment variable "HOUDINI_ADAPTOR_SERVER_PATH" to the socket the server is running on after the server has finished starting. """ @@ -175,7 +175,7 @@ def _start_houdini_server_thread(self) -> None: target=self._start_houdini_server, name="HoudiniAdaptorServerThread" ) self._server_thread.start() - os.environ["HOUDINI_ADAPTOR_SOCKET_PATH"] = self._wait_for_socket() + os.environ["HOUDINI_ADAPTOR_SERVER_PATH"] = self._wait_for_socket() @property def validators(self) -> AdaptorDataValidators: diff --git a/src/deadline/houdini_adaptor/HoudiniClient/houdini_client.py b/src/deadline/houdini_adaptor/HoudiniClient/houdini_client.py index c689627..63f8493 100644 --- a/src/deadline/houdini_adaptor/HoudiniClient/houdini_client.py +++ b/src/deadline/houdini_adaptor/HoudiniClient/houdini_client.py @@ -30,8 +30,8 @@ class HoudiniClient(HTTPClientInterface): Client that runs in Houdini for the Houdini Adaptor """ - def __init__(self, socket_path: str) -> None: - super().__init__(socket_path=socket_path) + def __init__(self, server_path: str) -> None: + super().__init__(server_path=server_path) self.actions.update(HoudiniHandler().action_dict) def close(self, args: Optional[dict] = None) -> None: @@ -42,21 +42,21 @@ def graceful_shutdown(self, signum: int, frame: FrameType | None): def main(): - socket_path = os.environ.get("HOUDINI_ADAPTOR_SOCKET_PATH") - if not socket_path: + server_path = os.environ.get("HOUDINI_ADAPTOR_SERVER_PATH") + if not server_path: raise OSError( "HoudiniClient cannot connect to the Adaptor because the environment variable " - "HOUDINI_ADAPTOR_SOCKET_PATH does not exist" + "HOUDINI_ADAPTOR_SERVER_PATH does not exist" ) - if not os.path.exists(socket_path): + if not os.path.exists(server_path): raise OSError( "HoudiniClient cannot connect to the Adaptor because the socket at the path defined by " - "the environment variable HOUDINI_ADAPTOR_SOCKET_PATH does not exist. Got: " - f"{os.environ['HOUDINI_ADAPTOR_SOCKET_PATH']}" + "the environment variable HOUDINI_ADAPTOR_SERVER_PATH does not exist. Got: " + f"{os.environ['HOUDINI_ADAPTOR_SERVER_PATH']}" ) - client = HoudiniClient(socket_path) + client = HoudiniClient(server_path) client.poll() diff --git a/src/deadline/houdini_submitter/otls/deadline_cloud.hda/Driver_1deadline__cloud/Tools.shelf b/src/deadline/houdini_submitter/otls/deadline_cloud.hda/Driver_1deadline__cloud/Tools.shelf index b3edbcc..8ad9cf3 100644 --- a/src/deadline/houdini_submitter/otls/deadline_cloud.hda/Driver_1deadline__cloud/Tools.shelf +++ b/src/deadline/houdini_submitter/otls/deadline_cloud.hda/Driver_1deadline__cloud/Tools.shelf @@ -12,7 +12,7 @@ $HDA_TABLE_AND_NAME - Digital Assets + Farm diff --git a/src/deadline/houdini_submitter/python/deadline_cloud_for_houdini/queue_parameters.py b/src/deadline/houdini_submitter/python/deadline_cloud_for_houdini/queue_parameters.py index 4dcb18c..1b48d0b 100644 --- a/src/deadline/houdini_submitter/python/deadline_cloud_for_houdini/queue_parameters.py +++ b/src/deadline/houdini_submitter/python/deadline_cloud_for_houdini/queue_parameters.py @@ -5,15 +5,11 @@ import hou +from ._version import version_tuple as adaptor_version_tuple from deadline.client.api._queue_parameters import get_queue_parameter_definitions from deadline.client.job_bundle.parameters import JobParameter -_QUEUE_ENVIRONMENT_SPECIAL_DEFAULTS = { - "RezPackages": "houdini deadline_cloud_for_houdini", -} - - def _get_queue_parameter_groups( queue_parameter_definitions: list[JobParameter], ) -> tuple[dict[str, list[JobParameter]], list[JobParameter]]: @@ -141,8 +137,13 @@ def _get_equivalent_bool(original_value: str) -> Optional[bool]: def _get_default_value(param: JobParameter) -> tuple[Union[str, int, float], ...]: - if param["name"] in _QUEUE_ENVIRONMENT_SPECIAL_DEFAULTS: - return (_QUEUE_ENVIRONMENT_SPECIAL_DEFAULTS[param["name"]],) + houdini_version = ".".join(hou.applicationVersionString().split(".")[:2]) + adaptor_version = ".".join(str(v) for v in adaptor_version_tuple[:2]) + + if param["name"] == "RezPackages": + return (f"houdini-{houdini_version}.* deadline_cloud_for_houdini",) + elif param["name"] == "CondaPackages": + return (f"houdini={houdini_version}.* houdini-openjd={adaptor_version}.*",) elif "default" in param: return (param["default"],) else: diff --git a/src/deadline/houdini_submitter/python/deadline_cloud_for_houdini/submitter.py b/src/deadline/houdini_submitter/python/deadline_cloud_for_houdini/submitter.py index d074f62..1a6e82e 100644 --- a/src/deadline/houdini_submitter/python/deadline_cloud_for_houdini/submitter.py +++ b/src/deadline/houdini_submitter/python/deadline_cloud_for_houdini/submitter.py @@ -8,9 +8,6 @@ from pathlib import Path from deadline.client.job_bundle._yaml import deadline_yaml_dump -from deadline.client.job_bundle.adaptors import ( - parse_frame_range, -) from deadline.client import api from deadline.client.job_bundle.submission import AssetReferences from deadline.client.job_bundle import create_job_history_bundle_dir @@ -148,6 +145,7 @@ def _get_parameter_values(node: hou.Node) -> dict[str, Any]: {"name": "deadline:targetTaskRunStatus", "value": initial_status}, {"name": "deadline:maxFailedTasksCount", "value": failed_tasks_limit}, {"name": "deadline:maxRetriesPerTask", "value": task_retry_limit}, + {"name": "HipFile", "value": _get_hip_file()}, *get_queue_parameter_values_as_openjd(node), ] } @@ -200,13 +198,11 @@ def _get_job_template(rop: hou.Node) -> dict[str, Any]: task_data_contents.append("frame: {{Task.Param.Frame}}\n") task_data_contents.append("ignore_input_nodes: true\n") # step - frame_list = "{start}-{stop}:{step}".format(**n) + frame_range = "{start}-{stop}:{step}".format(**n) step = { "name": n["name"], "parameterSpace": { - "taskParameterDefinitions": [ - {"name": "Frame", "range": parse_frame_range(frame_list), "type": "INT"} - ] + "taskParameterDefinitions": [{"name": "Frame", "range": frame_range, "type": "INT"}] }, "stepEnvironments": environments, "script": { diff --git a/test/deadline_adaptor_for_houdini/unit/HoudiniAdaptor/test_adaptor.py b/test/deadline_adaptor_for_houdini/unit/HoudiniAdaptor/test_adaptor.py index abf5b84..84146ba 100644 --- a/test/deadline_adaptor_for_houdini/unit/HoudiniAdaptor/test_adaptor.py +++ b/test/deadline_adaptor_for_houdini/unit/HoudiniAdaptor/test_adaptor.py @@ -66,7 +66,7 @@ def test_no_error( ) -> None: """Tests that on_start completes without error""" adaptor = HoudiniAdaptor(init_data) - mock_server.return_value.socket_path = "/tmp/9999" + mock_server.return_value.server_path = "/tmp/9999" adaptor.on_start() @patch("time.sleep") @@ -87,7 +87,7 @@ def test_waits_for_server_socket( socket_mock = PropertyMock( side_effect=[None, None, None, "/tmp/9999", "/tmp/9999", "/tmp/9999"] ) - type(mock_server.return_value).socket_path = socket_mock + type(mock_server.return_value).server_path = socket_mock # WHEN adaptor.on_start() @@ -158,7 +158,7 @@ def test_houdini_init_timeout( """ # GIVEN adaptor = HoudiniAdaptor(init_data) - mock_server.return_value.socket_path = "/tmp/9999" + mock_server.return_value.server_path = "/tmp/9999" new_timeout = 0.01 with patch.object(adaptor, "_HOUDINI_START_TIMEOUT_SECONDS", new_timeout), pytest.raises( @@ -190,7 +190,7 @@ def test_houdini_init_fail( """ # GIVEN adaptor = HoudiniAdaptor(init_data) - mock_server.return_value.socket_path = "/tmp/9999" + mock_server.return_value.server_path = "/tmp/9999" with pytest.raises(RuntimeError) as exc_info: # WHEN @@ -216,7 +216,7 @@ def test_populate_action_queue( # GIVEN mock_actions_queue.__len__.return_value = 0 adaptor = HoudiniAdaptor(init_data) - mock_server.return_value.socket_path = "/tmp/9999" + mock_server.return_value.server_path = "/tmp/9999" # WHEN adaptor.on_start() @@ -254,7 +254,7 @@ def test_populate_action_queue_less_init_data( missing_field = "scene_file" mock_actions_queue.__len__.return_value = 0 adaptor = HoudiniAdaptor(init_data) - mock_server.return_value.socket_path = "/tmp/9999" + mock_server.return_value.server_path = "/tmp/9999" # WHEN try: @@ -283,7 +283,7 @@ def test_on_run( """Tests that on_run waits for completion""" # GIVEN adaptor = HoudiniAdaptor(init_data) - mock_server.return_value.socket_path = "/tmp/9999" + mock_server.return_value.server_path = "/tmp/9999" # First side_effect value consumed by setter is_rendering_mock = PropertyMock(side_effect=[None, True, False]) HoudiniAdaptor._is_rendering = is_rendering_mock @@ -324,7 +324,7 @@ def test_on_run_render_fail( mock_houdini_is_running.side_effect = [True, True, True, False, False] mock_logging_subprocess.return_value.returncode = 1 adaptor = HoudiniAdaptor(init_data) - mock_server.return_value.socket_path = "/tmp/9999" + mock_server.return_value.server_path = "/tmp/9999" adaptor.on_start() # WHEN @@ -355,7 +355,7 @@ def test_on_stop( ) -> None: # GIVEN adaptor = HoudiniAdaptor(init_data) - mock_server.return_value.socket_path = "/tmp/9999" + mock_server.return_value.server_path = "/tmp/9999" is_rendering_mock = PropertyMock(return_value=False) HoudiniAdaptor._is_rendering = is_rendering_mock adaptor.on_start() @@ -437,7 +437,7 @@ def test_on_cleanup( ) -> None: # GIVEN adaptor = HoudiniAdaptor(init_data) - mock_server.return_value.socket_path = "/tmp/9999" + mock_server.return_value.server_path = "/tmp/9999" is_rendering_mock = PropertyMock(return_value=False) HoudiniAdaptor._is_rendering = is_rendering_mock diff --git a/test/deadline_adaptor_for_houdini/unit/HoudiniClient/test_client.py b/test/deadline_adaptor_for_houdini/unit/HoudiniClient/test_client.py index aaf3010..ac9286c 100644 --- a/test/deadline_adaptor_for_houdini/unit/HoudiniClient/test_client.py +++ b/test/deadline_adaptor_for_houdini/unit/HoudiniClient/test_client.py @@ -15,11 +15,11 @@ class TestHoudiniClient: @patch("deadline.houdini_adaptor.HoudiniClient.houdini_client.HTTPClientInterface") def test_houdiniclient(self, mock_httpclient: Mock) -> None: """Tests that the houdini client can initialize, set a renderer and close""" - client = HoudiniClient(socket_path=str(9999)) + client = HoudiniClient(server_path=str(9999)) client.close() @patch("deadline.houdini_adaptor.HoudiniClient.houdini_client.os.path.exists") - @patch.dict(os.environ, {"HOUDINI_ADAPTOR_SOCKET_PATH": "socket_path"}) + @patch.dict(os.environ, {"HOUDINI_ADAPTOR_SERVER_PATH": "server_path"}) @patch("deadline.houdini_adaptor.HoudiniClient.HoudiniClient.poll") @patch("deadline.houdini_adaptor.HoudiniClient.houdini_client.HTTPClientInterface") def test_main(self, mock_httpclient: Mock, mock_poll: Mock, mock_exists: Mock) -> None: @@ -31,7 +31,7 @@ def test_main(self, mock_httpclient: Mock, mock_poll: Mock, mock_exists: Mock) - main() # THEN - mock_exists.assert_called_once_with("socket_path") + mock_exists.assert_called_once_with("server_path") mock_poll.assert_called_once() @patch.dict(os.environ, {}, clear=True) @@ -45,11 +45,11 @@ def test_main_no_server_socket(self, mock_poll: Mock) -> None: # THEN assert str(exc_info.value) == ( "HoudiniClient cannot connect to the Adaptor because the environment variable " - "HOUDINI_ADAPTOR_SOCKET_PATH does not exist" + "HOUDINI_ADAPTOR_SERVER_PATH does not exist" ) mock_poll.assert_not_called() - @patch.dict(os.environ, {"HOUDINI_ADAPTOR_SOCKET_PATH": "/a/path/that/does/not/exist"}) + @patch.dict(os.environ, {"HOUDINI_ADAPTOR_SERVER_PATH": "/a/path/that/does/not/exist"}) @patch("deadline.houdini_adaptor.HoudiniClient.houdini_client.os.path.exists") @patch("deadline.houdini_adaptor.HoudiniClient.HoudiniClient.poll") def test_main_server_socket_not_exists(self, mock_poll: Mock, mock_exists: Mock) -> None: @@ -62,10 +62,10 @@ def test_main_server_socket_not_exists(self, mock_poll: Mock, mock_exists: Mock) main() # THEN - mock_exists.assert_called_once_with(os.environ["HOUDINI_ADAPTOR_SOCKET_PATH"]) + mock_exists.assert_called_once_with(os.environ["HOUDINI_ADAPTOR_SERVER_PATH"]) assert str(exc_info.value) == ( "HoudiniClient cannot connect to the Adaptor because the socket at the path defined by " - "the environment variable HOUDINI_ADAPTOR_SOCKET_PATH does not exist. Got: " - f"{os.environ['HOUDINI_ADAPTOR_SOCKET_PATH']}" + "the environment variable HOUDINI_ADAPTOR_SERVER_PATH does not exist. Got: " + f"{os.environ['HOUDINI_ADAPTOR_SERVER_PATH']}" ) mock_poll.assert_not_called()