From 7c4e2b8b1aeadf614f3d5d61a8b2c25176493de3 Mon Sep 17 00:00:00 2001 From: Lucas Eckhardt <117225985+lucaseck@users.noreply.github.com> Date: Mon, 12 Feb 2024 10:20:41 -0600 Subject: [PATCH] feat: add install builder component (#5) Signed-off-by: Lucas Eckhardt <117225985+lucaseck@users.noreply.github.com> --- depsBundle.sh | 19 +++ .../deadline-cloud-for-keyshot.xml | 146 ++++++++++++++++++ ...Submitter.py => DeadlineCloudSubmitter.py} | 10 ++ scripts/_project.py | 82 ++++++++++ scripts/deps_bundle.py | 133 ++++++++++++++++ 5 files changed, 390 insertions(+) create mode 100755 depsBundle.sh create mode 100644 install_builder/deadline-cloud-for-keyshot.xml rename keyshot_script/{DealineCloudSubmitter.py => DeadlineCloudSubmitter.py} (76%) create mode 100644 scripts/_project.py create mode 100644 scripts/deps_bundle.py diff --git a/depsBundle.sh b/depsBundle.sh new file mode 100755 index 0000000..736e5f8 --- /dev/null +++ b/depsBundle.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +set -xeuo pipefail + +SCRIPT_FOLDER=$(dirname "$0")/scripts + +pushd "${SCRIPT_FOLDER}" +python deps_bundle.py +popd + +rm -f dependency_bundle/deadline_cloud_for_keyshot_submitter-deps-windows.zip +rm -f dependency_bundle/deadline_cloud_for_keyshot_submitter-deps-linux.zip +rm -f dependency_bundle/deadline_cloud_for_keyshot_submitter-deps-macos.zip + +mkdir -p dependency_bundle + +cp scripts/dependency_bundle/deadline_cloud_for_keyshot_submitter-deps.zip dependency_bundle/deadline_cloud_for_keyshot_submitter-deps-windows.zip +cp scripts/dependency_bundle/deadline_cloud_for_keyshot_submitter-deps.zip dependency_bundle/deadline_cloud_for_keyshot_submitter-deps-linux.zip +cp scripts/dependency_bundle/deadline_cloud_for_keyshot_submitter-deps.zip dependency_bundle/deadline_cloud_for_keyshot_submitter-deps-macos.zip \ No newline at end of file diff --git a/install_builder/deadline-cloud-for-keyshot.xml b/install_builder/deadline-cloud-for-keyshot.xml new file mode 100644 index 0000000..9c1e5e8 --- /dev/null +++ b/install_builder/deadline-cloud-for-keyshot.xml @@ -0,0 +1,146 @@ + + deadline_cloud_for_keyshot + Deadline Cloud for KeyShot 12 + KeyShot plugin for submitting jobs to Amazon Deadline Cloud. + 1 + 0 + 1 + + + KeyShot Plug-in Script + ${keyshot_scripts_folder} + keyshotplugin + all + + + components/deadline-cloud-for-keyshot/keyshot_script/DeadlineCloudSubmitter.py + + + + + KeyShot Submitter Files + ${keyshot_installdir}/keyshot_submitter + keyshot + all + + + components/deadline-cloud-for-keyshot/src/deadline/keyshot_submitter/* + + + + + Dependency Files + ${installdir}/tmp/keyshot_deps + keyshotdeps + all + + + components/deadline-cloud-for-keyshot/dependency_bundle + + + + + + + + + does_not_contain + ${platform_name} + linux + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Deadline Cloud for KeyShot +- Install the integrated KeyShot submitter files to the installation directory +- Register the plug-in with KeyShot by moving the DeadlineCloudSubmitter script to the KeyShot scripts folder +- Sets the DEADLINE_KEYSHOT environment variable to point the DeadlineCloudSubmitter script to the submitter module + + + + + + ${keyshot_installdir} + ${installdir}/tmp/keyshot_deps/dependency_bundle/deadline_cloud_for_keyshot_submitter-deps-${keyshot_deps_platform}.zip + + + ${installdir}/tmp/keyshot_deps + + + DEADLINE_KEYSHOT + ${keyshot_installdir}/keyshot_submitter + ${installscope} + + + + + does_not_contain + ${platform_name} + linux + + + \ No newline at end of file diff --git a/keyshot_script/DealineCloudSubmitter.py b/keyshot_script/DeadlineCloudSubmitter.py similarity index 76% rename from keyshot_script/DealineCloudSubmitter.py rename to keyshot_script/DeadlineCloudSubmitter.py index eb93977..29c478b 100644 --- a/keyshot_script/DealineCloudSubmitter.py +++ b/keyshot_script/DeadlineCloudSubmitter.py @@ -18,6 +18,16 @@ # In dev mode: src/deadline/keyshot_submitter DEADLINE_KEYSHOT = os.getenv("DEADLINE_KEYSHOT") +if not DEADLINE_PYTHON: + raise RuntimeError( + "Environment variable DEADLINE_PYTHON not set. Please set DEADLINE_PYTHON to point to an installed version of Python with Pyside2." + ) + +if not DEADLINE_KEYSHOT: + raise RuntimeError( + "Environment variable DEADLINE_KEYSHOT not set. Please set DEADLINE_KEYSHOT to point to the keyshot_submitter folder." + ) + # save scene information to json file for submitter module to load scene_info = lux.getSceneInfo() opts = lux.getRenderOptions() diff --git a/scripts/_project.py b/scripts/_project.py new file mode 100644 index 0000000..11b13b9 --- /dev/null +++ b/scripts/_project.py @@ -0,0 +1,82 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +from __future__ import annotations + +import subprocess +import sys + +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Any, Optional + + +ADAPTOR_ONLY_DEPENDENCIES = {"openjd-adaptor-runtime"} + + +def get_project_dict(project_path: Optional[Path] = None) -> dict[str, Any]: + if sys.version_info < (3, 11): + with TemporaryDirectory() as toml_env: + toml_install_pip_args = ["pip", "install", "--target", toml_env, "toml"] + subprocess.run(toml_install_pip_args, check=True) + sys.path.insert(0, toml_env) + import toml + mode = "r" + else: + import tomllib as toml + + mode = "rb" + + with open(str((project_path or get_git_root()) / "pyproject.toml"), mode) as pyproject_toml: + return toml.load(pyproject_toml) + + +class Dependency: + name: str + operator: Optional[str] + version: Optional[str] + + def __init__(self, dep: str): + components = dep.split(" ") + self.name = components[0] + if len(components) > 2: + self.operator = components[1] + self.version = components[2] + else: + self.operator = None + self.version = None + + def for_pip(self) -> str: + if self.operator is not None and self.version is not None: + return f"{self.name}{self.operator}{self.version}" + return self.name + + def __repr__(self) -> str: + return self.for_pip() + + +def get_dependencies(pyproject_dict: dict[str, Any], exclude_adaptor_only=True) -> list[Dependency]: + if "project" not in pyproject_dict: + raise Exception("pyproject.toml is missing project section") + if "dependencies" not in pyproject_dict["project"]: + raise Exception("pyproject.toml is missing dependencies section") + + return [ + Dependency(dep_str) + for dep_str in pyproject_dict["project"]["dependencies"] + if exclude_adaptor_only or dep_str not in ADAPTOR_ONLY_DEPENDENCIES + ] + + +def get_git_root() -> Path: + return Path(__file__).parents[1].resolve() + + +def get_pip_platform(system_platform: str) -> str: + if system_platform == "Windows": + return "win_amd64" + elif system_platform == "Darwin": + return "macosx_10_9_x86_64" + elif system_platform == "Linux": + return "manylinux2014_x86_64" + else: + raise Exception(f"Unsupported platform: {system_platform}") diff --git a/scripts/deps_bundle.py b/scripts/deps_bundle.py new file mode 100644 index 0000000..646c8f4 --- /dev/null +++ b/scripts/deps_bundle.py @@ -0,0 +1,133 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +from __future__ import annotations + +import re +import shutil +import subprocess + +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Any + +from _project import get_project_dict, get_dependencies, get_pip_platform, Dependency + +SUPPORTED_PYTHON_VERSIONS = ["3.9", "3.10", "3.11"] +SUPPORTED_PLATFORMS = ["Windows", "Linux", "Darwin"] +NATIVE_DEPENDENCIES = ["xxhash"] + + +def _get_package_version_regex(package: str) -> re.Pattern: + return re.compile(rf"^{re.escape(package)} *(.*)$") + + +def _get_package_version(package: str, install_path: Path) -> str: + version_regex = _get_package_version_regex(package) + pip_args = ["pip", "list", "--path", str(install_path)] + output = subprocess.run(pip_args, check=True, capture_output=True).stdout.decode("utf-8") + for line in output.split("\n"): + match = version_regex.match(line) + if match: + return match.group(1) + raise Exception(f"Could not find version for package {package}") + + +def _build_base_environment(working_directory: Path, dependencies: list[Dependency]) -> Path: + (working_directory / "base_env").mkdir() + base_env_path = working_directory / "base_env" + dependencies_for_pip = [d.for_pip() for d in dependencies] + base_env_pip_args = [ + "pip", + "install", + "--target", + str(base_env_path), + "--only-binary=:all:", + *dependencies_for_pip, + ] + subprocess.run(base_env_pip_args, check=True) + return base_env_path + + +def _download_native_dependencies(working_directory: Path, base_env: Path) -> list[Path]: + versioned_native_dependencies = [ + f"{package_name}=={_get_package_version(package_name, base_env)}" + for package_name in NATIVE_DEPENDENCIES + ] + native_dependency_paths = [] + for version in SUPPORTED_PYTHON_VERSIONS: + for plat in map(get_pip_platform, SUPPORTED_PLATFORMS): + native_dependency_path = ( + working_directory / "native" / f"{version.replace('.', '_')}_{plat}" + ) + native_dependency_paths.append(native_dependency_path) + native_dependency_path.mkdir(parents=True) + native_dependency_pip_args = [ + "pip", + "install", + "--target", + str(native_dependency_path), + "--platform", + plat, + "--python-version", + version, + "--only-binary=:all:", + *versioned_native_dependencies, + ] + subprocess.run(native_dependency_pip_args, check=True) + return native_dependency_paths + + +def _copy_native_to_base_env(base_env: Path, native_dependency_paths: list[Path]) -> None: + for native_dependency_path in native_dependency_paths: + for file in native_dependency_path.rglob("*"): + if file.is_file(): + relative = file.relative_to(native_dependency_path) + in_base_env = base_env / relative + if not in_base_env.exists(): + in_base_env.parent.mkdir(parents=True, exist_ok=True) + shutil.copy(str(file), str(in_base_env)) + + +def _get_zip_path(working_directory: Path, project_dict: dict[str, Any]) -> Path: + if "project" not in project_dict: + raise Exception("pyproject.toml is missing project section") + if "name" not in project_dict["project"]: + raise Exception("pyproject.toml is missing name section") + transformed_project_name = ( + f"{project_dict['project']['name'].replace('-', '_')}_submitter-deps.zip" + ) + return working_directory / transformed_project_name + + +def _zip_bundle(base_env: Path, zip_path: Path) -> None: + shutil.make_archive(str(zip_path.with_suffix("")), "zip", str(base_env)) + + +def _copy_zip_to_destination(zip_path: Path) -> None: + dependency_bundle_dir = Path.cwd() / "dependency_bundle" + dependency_bundle_dir.mkdir(exist_ok=True) + zip_destination = dependency_bundle_dir / zip_path.name + if zip_destination.exists(): + zip_destination.unlink() + shutil.copy(str(zip_path), str(zip_destination)) + + +def build_deps_bundle() -> None: + with TemporaryDirectory() as working_directory: + working_directory = Path(working_directory) + project_dict = get_project_dict() + dependencies: list[Dependency] = get_dependencies(project_dict) + deps_noopenjd: list[Dependency] = filter( + lambda dep: not dep.name.startswith("openjd"), dependencies + ) + base_env = _build_base_environment(working_directory, deps_noopenjd) + native_dependency_paths = _download_native_dependencies(working_directory, base_env) + _copy_native_to_base_env(base_env, native_dependency_paths) + zip_path = _get_zip_path(working_directory, project_dict) + _zip_bundle(base_env, zip_path) + print(list(working_directory.glob("*"))) + _copy_zip_to_destination(zip_path) + + +if __name__ == "__main__": + build_deps_bundle()