From 43ef52306afb620831388014fab94a0b738fe66c Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 18 Jun 2024 15:51:09 +0200 Subject: [PATCH 1/4] Merge branch 'develop' of https://github.com/ynput/ayon-core into enhancement/usd_workflow # Conflicts: # server_addon/deadline/client/ayon_deadline/plugins/publish/submit_publish_job.py # server_addon/houdini/client/ayon_houdini/version.py # server_addon/houdini/package.py --- .../publish/submit_houdini_render_deadline.py | 96 +++++++++++++++++-- .../plugins/publish/submit_publish_job.py | 2 +- 2 files changed, 89 insertions(+), 9 deletions(-) diff --git a/client/ayon_deadline/plugins/publish/submit_houdini_render_deadline.py b/client/ayon_deadline/plugins/publish/submit_houdini_render_deadline.py index d3fbe62963..7956108e77 100644 --- a/client/ayon_deadline/plugins/publish/submit_houdini_render_deadline.py +++ b/client/ayon_deadline/plugins/publish/submit_houdini_render_deadline.py @@ -50,6 +50,30 @@ class RedshiftRenderPluginInfo(): Version = attr.ib(default="1") +@attr.s +class HuskStandalonePluginInfo(): + """Requires Deadline Husk Standalone Plugin. + See Deadline Plug-in: + https://github.com/BigRoy/HuskStandaloneSubmitter + Also see Husk options here: + https://www.sidefx.com/docs/houdini/ref/utils/husk.html + """ + SceneFile = attr.ib() + # TODO: Below parameters are only supported by custom version of the plugin + Renderer = attr.ib(default=None) + RenderSettings = attr.ib(default="/Render/rendersettings") + Purpose = attr.ib(default="geometry,render") + Complexity = attr.ib(default="veryhigh") + Snapshot = attr.ib(default=-1) + LogLevel = attr.ib(default="2") + PreRender = attr.ib(default="") + PreFrame = attr.ib(default="") + PostFrame = attr.ib(default="") + PostRender = attr.ib(default="") + RestartDelegate = attr.ib(default="") + Version = attr.ib(default="") + + class HoudiniSubmitDeadline( abstract_submit_deadline.AbstractSubmitDeadline, AYONPyblishPluginMixin @@ -69,8 +93,7 @@ class HoudiniSubmitDeadline( label = "Submit Render to Deadline" order = pyblish.api.IntegratorOrder hosts = ["houdini"] - families = ["usdrender", - "redshift_rop", + families = ["redshift_rop", "arnold_rop", "mantra_rop", "karma_rop", @@ -149,11 +172,14 @@ def get_job_info(self, dependency_job_ids=None): job_type = "[RENDER]" if split_render_job and not is_export_job: - # Convert from family to Deadline plugin name - # i.e., arnold_rop -> Arnold - plugin = ( - instance.data["productType"].replace("_rop", "").capitalize() - ) + product_type = instance.data["productType"] + plugin = { + "usdrender": "HuskStandalone", + }.get(product_type) + if not plugin: + # Convert from product type to Deadline plugin name + # i.e., arnold_rop -> Arnold + plugin = product_type.replace("_rop", "").capitalize() else: plugin = "Houdini" if split_render_job: @@ -185,7 +211,8 @@ def get_job_info(self, dependency_job_ids=None): # Make sure we make job frame dependent so render tasks pick up a soon # as export tasks are done if split_render_job and not is_export_job: - job_info.IsFrameDependent = True + job_info.IsFrameDependent = bool(instance.data.get( + "splitRenderFrameDependent", True)) job_info.Pool = instance.data.get("primaryPool") job_info.SecondaryPool = instance.data.get("secondaryPool") @@ -207,6 +234,13 @@ def get_job_info(self, dependency_job_ids=None): ) job_info.Group = self.group + # Apply render globals, like e.g. data from collect machine list + render_globals = instance.data.get("renderGlobals", {}) + if render_globals: + self.log.debug("Applying 'renderGlobals' to job info: %s", + render_globals) + job_info.update(render_globals) + job_info.Comment = context.data.get("comment") keys = [ @@ -292,6 +326,10 @@ def get_plugin_info(self, job_type=None): " - using version configured in Deadline" )) + elif product_type == "usdrender": + plugin_info = self._get_husk_standalone_plugin_info( + instance, hou_major_minor) + else: self.log.error( "Product type '%s' not supported yet to split render job", @@ -321,3 +359,45 @@ def process(self, instance): # Store output dir for unified publisher (filesequence) output_dir = os.path.dirname(instance.data["files"][0]) instance.data["outputDir"] = output_dir + + def _get_husk_standalone_plugin_info(self, instance, hou_major_minor): + # Not all hosts can import this module. + import hou + + # Supply additional parameters from the USD Render ROP + # to the Husk Standalone Render Plug-in + rop_node = hou.node(instance.data["instance_node"]) + snapshot_interval = -1 + if rop_node.evalParm("dosnapshot"): + snapshot_interval = rop_node.evalParm("snapshotinterval") + + restart_delegate = 0 + if rop_node.evalParm("husk_restartdelegate"): + restart_delegate = rop_node.evalParm("husk_restartdelegateframes") + + rendersettings = ( + rop_node.evalParm("rendersettings") + or "/Render/rendersettings" + ) + return HuskStandalonePluginInfo( + SceneFile=instance.data["ifdFile"], + Renderer=rop_node.evalParm("renderer"), + RenderSettings=rendersettings, + Purpose=rop_node.evalParm("husk_purpose"), + Complexity=rop_node.evalParm("husk_complexity"), + Snapshot=snapshot_interval, + PreRender=rop_node.evalParm("husk_prerender"), + PreFrame=rop_node.evalParm("husk_preframe"), + PostFrame=rop_node.evalParm("husk_postframe"), + PostRender=rop_node.evalParm("husk_postrender"), + RestartDelegate=restart_delegate, + Version=hou_major_minor + ) + + +class HoudiniSubmitDeadlineUsdRender(HoudiniSubmitDeadline): + # Do not use published workfile paths for USD Render ROP because the + # Export Job doesn't seem to occur using the published path either, so + # output paths then do not match the actual rendered paths + use_published = False + families = ["usdrender"] diff --git a/client/ayon_deadline/plugins/publish/submit_publish_job.py b/client/ayon_deadline/plugins/publish/submit_publish_job.py index 721cd682ea..6aafb8f1c6 100644 --- a/client/ayon_deadline/plugins/publish/submit_publish_job.py +++ b/client/ayon_deadline/plugins/publish/submit_publish_job.py @@ -94,7 +94,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin, "vrayscene", "maxrender", "arnold_rop", "mantra_rop", "karma_rop", "vray_rop", - "redshift_rop"] + "redshift_rop", "usdrender"] settings_category = "deadline" aov_filter = [ From 92a211488562f8294e7d05749796159221ce2566 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 3 Jul 2024 17:56:38 +0200 Subject: [PATCH 2/4] Standardize IS_TEST to AYON_IN_TESTS ayon-core/client/ayon_core/lib/ayon_info.py expects AYON_IN_TESTS which is more descriptive than just IS_TEST --- .../plugins/publish/submit_aftereffects_deadline.py | 2 +- .../plugins/publish/submit_blender_deadline.py | 2 +- .../ayon_deadline/plugins/publish/submit_fusion_deadline.py | 2 +- .../plugins/publish/submit_harmony_deadline.py | 2 +- client/ayon_deadline/plugins/publish/submit_max_deadline.py | 2 +- .../ayon_deadline/plugins/publish/submit_maya_deadline.py | 6 +++--- .../plugins/publish/submit_publish_cache_job.py | 2 +- client/ayon_deadline/plugins/publish/submit_publish_job.py | 2 +- .../repository/custom/plugins/GlobalJobPreLoad.py | 6 +++--- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/client/ayon_deadline/plugins/publish/submit_aftereffects_deadline.py b/client/ayon_deadline/plugins/publish/submit_aftereffects_deadline.py index a7be38bf3b..45d907cbba 100644 --- a/client/ayon_deadline/plugins/publish/submit_aftereffects_deadline.py +++ b/client/ayon_deadline/plugins/publish/submit_aftereffects_deadline.py @@ -87,7 +87,7 @@ def get_job_info(self): "AYON_WORKDIR", "AYON_APP_NAME", "AYON_LOG_NO_COLORS", - "IS_TEST" + "AYON_IN_TESTS" ] environment = { diff --git a/client/ayon_deadline/plugins/publish/submit_blender_deadline.py b/client/ayon_deadline/plugins/publish/submit_blender_deadline.py index 479dffdcc4..073de909b4 100644 --- a/client/ayon_deadline/plugins/publish/submit_blender_deadline.py +++ b/client/ayon_deadline/plugins/publish/submit_blender_deadline.py @@ -110,7 +110,7 @@ def get_job_info(self): "AYON_TASK_NAME", "AYON_WORKDIR", "AYON_APP_NAME", - "IS_TEST" + "AYON_IN_TESTS" ] environment = { diff --git a/client/ayon_deadline/plugins/publish/submit_fusion_deadline.py b/client/ayon_deadline/plugins/publish/submit_fusion_deadline.py index f90bb9b597..bf9df40edc 100644 --- a/client/ayon_deadline/plugins/publish/submit_fusion_deadline.py +++ b/client/ayon_deadline/plugins/publish/submit_fusion_deadline.py @@ -217,7 +217,7 @@ def process(self, instance): "AYON_WORKDIR", "AYON_APP_NAME", "AYON_LOG_NO_COLORS", - "IS_TEST", + "AYON_IN_TESTS", "AYON_BUNDLE_NAME", ] diff --git a/client/ayon_deadline/plugins/publish/submit_harmony_deadline.py b/client/ayon_deadline/plugins/publish/submit_harmony_deadline.py index 1a40f6f302..bc91483c4f 100644 --- a/client/ayon_deadline/plugins/publish/submit_harmony_deadline.py +++ b/client/ayon_deadline/plugins/publish/submit_harmony_deadline.py @@ -282,7 +282,7 @@ def get_job_info(self): "AYON_WORKDIR", "AYON_APP_NAME", "AYON_LOG_NO_COLORS" - "IS_TEST" + "AYON_IN_TESTS" ] environment = { diff --git a/client/ayon_deadline/plugins/publish/submit_max_deadline.py b/client/ayon_deadline/plugins/publish/submit_max_deadline.py index a287630dc5..6a369eb001 100644 --- a/client/ayon_deadline/plugins/publish/submit_max_deadline.py +++ b/client/ayon_deadline/plugins/publish/submit_max_deadline.py @@ -114,7 +114,7 @@ def get_job_info(self): "AYON_TASK_NAME", "AYON_WORKDIR", "AYON_APP_NAME", - "IS_TEST", + "AYON_IN_TESTS", ] environment = { diff --git a/client/ayon_deadline/plugins/publish/submit_maya_deadline.py b/client/ayon_deadline/plugins/publish/submit_maya_deadline.py index beed19e007..d50b0147d9 100644 --- a/client/ayon_deadline/plugins/publish/submit_maya_deadline.py +++ b/client/ayon_deadline/plugins/publish/submit_maya_deadline.py @@ -216,7 +216,7 @@ def get_job_info(self): "AYON_TASK_NAME", "AYON_WORKDIR", "AYON_APP_NAME", - "IS_TEST" + "AYON_IN_TESTS" ] environment = { @@ -236,7 +236,7 @@ def get_job_info(self): job_info.EnvironmentKeyValue["AYON_LOG_NO_COLORS"] = "1" # Adding file dependencies. - if not bool(os.environ.get("IS_TEST")) and self.asset_dependencies: + if not is_in_tests() and self.asset_dependencies: dependencies = instance.context.data["fileDependencies"] for dependency in dependencies: job_info.AssetDependency += dependency @@ -589,7 +589,7 @@ def _get_maya_payload(self, data): job_info = copy.deepcopy(self.job_info) - if not bool(os.environ.get("IS_TEST")) and self.asset_dependencies: + if not is_in_tests() and self.asset_dependencies: # Asset dependency to wait for at least the scene file to sync. job_info.AssetDependency += self.scene_path diff --git a/client/ayon_deadline/plugins/publish/submit_publish_cache_job.py b/client/ayon_deadline/plugins/publish/submit_publish_cache_job.py index 2bb138d849..ddfd21dc78 100644 --- a/client/ayon_deadline/plugins/publish/submit_publish_cache_job.py +++ b/client/ayon_deadline/plugins/publish/submit_publish_cache_job.py @@ -128,7 +128,7 @@ def _submit_deadline_post_job(self, instance, job): "AYON_TASK_NAME": instance.context.data["task"], "AYON_USERNAME": instance.context.data["user"], "AYON_LOG_NO_COLORS": "1", - "IS_TEST": str(int(is_in_tests())), + "AYON_IN_TESTS": str(int(is_in_tests())), "AYON_PUBLISH_JOB": "1", "AYON_RENDER_JOB": "0", "AYON_REMOTE_PUBLISH": "0", diff --git a/client/ayon_deadline/plugins/publish/submit_publish_job.py b/client/ayon_deadline/plugins/publish/submit_publish_job.py index 6aafb8f1c6..11bdc461f6 100644 --- a/client/ayon_deadline/plugins/publish/submit_publish_job.py +++ b/client/ayon_deadline/plugins/publish/submit_publish_job.py @@ -206,7 +206,7 @@ def _submit_deadline_post_job(self, instance, job, instances): "AYON_TASK_NAME": instance.context.data["task"], "AYON_USERNAME": instance.context.data["user"], "AYON_LOG_NO_COLORS": "1", - "IS_TEST": str(int(is_in_tests())), + "AYON_IN_TESTS": str(int(is_in_tests())), "AYON_PUBLISH_JOB": "1", "AYON_RENDER_JOB": "0", "AYON_REMOTE_PUBLISH": "0", diff --git a/client/ayon_deadline/repository/custom/plugins/GlobalJobPreLoad.py b/client/ayon_deadline/repository/custom/plugins/GlobalJobPreLoad.py index ac04407f5b..cb34d4aa40 100644 --- a/client/ayon_deadline/repository/custom/plugins/GlobalJobPreLoad.py +++ b/client/ayon_deadline/repository/custom/plugins/GlobalJobPreLoad.py @@ -13,7 +13,7 @@ FileUtils, DirectoryUtils, ) -__version__ = "1.1.0" +__version__ = "1.1.1" VERSION_REGEX = re.compile( r"(?P0|[1-9]\d*)" r"\.(?P0|[1-9]\d*)" @@ -342,7 +342,7 @@ def inject_openpype_environment(deadlinePlugin): "envgroup": "farm" } - if job.GetJobEnvironmentKeyValue("IS_TEST"): + if job.GetJobEnvironmentKeyValue("AYON_IN_TESTS"): args.append("--automatic-tests") if all(add_kwargs.values()): @@ -501,7 +501,7 @@ def inject_ayon_environment(deadlinePlugin): "extractenvironments", export_url ] - if job.GetJobEnvironmentKeyValue("IS_TEST"): + if job.GetJobEnvironmentKeyValue("AYON_IN_TESTS"): args.append("--automatic-tests") for key, value in add_kwargs.items(): From c1fc19482d8bbcfd79882f5d8c94966afb2bcbfa Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 3 Jul 2024 18:00:12 +0200 Subject: [PATCH 3/4] Use legacy IS_TEST env var to mark automatic tests for OP --- .../repository/custom/plugins/GlobalJobPreLoad.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/ayon_deadline/repository/custom/plugins/GlobalJobPreLoad.py b/client/ayon_deadline/repository/custom/plugins/GlobalJobPreLoad.py index cb34d4aa40..4081f02015 100644 --- a/client/ayon_deadline/repository/custom/plugins/GlobalJobPreLoad.py +++ b/client/ayon_deadline/repository/custom/plugins/GlobalJobPreLoad.py @@ -342,7 +342,8 @@ def inject_openpype_environment(deadlinePlugin): "envgroup": "farm" } - if job.GetJobEnvironmentKeyValue("AYON_IN_TESTS"): + # use legacy IS_TEST env var to mark automatic tests for OP + if job.GetJobEnvironmentKeyValue("IS_TEST"): args.append("--automatic-tests") if all(add_kwargs.values()): From 251973ae082dcaef9bb1cb91c3acf45b4bb7419d Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 Jul 2024 12:25:42 +0200 Subject: [PATCH 4/4] Use only AYON_IN_TESTS to drive automatic tests --automatic-tests is removed in AYON --- .../plugins/publish/submit_publish_cache_job.py | 3 --- client/ayon_deadline/plugins/publish/submit_publish_job.py | 3 --- .../repository/custom/plugins/GlobalJobPreLoad.py | 6 ++++-- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/client/ayon_deadline/plugins/publish/submit_publish_cache_job.py b/client/ayon_deadline/plugins/publish/submit_publish_cache_job.py index ddfd21dc78..d93592a6a3 100644 --- a/client/ayon_deadline/plugins/publish/submit_publish_cache_job.py +++ b/client/ayon_deadline/plugins/publish/submit_publish_cache_job.py @@ -156,9 +156,6 @@ def _submit_deadline_post_job(self, instance, job): "--targets", "farm" ] - if is_in_tests(): - args.append("--automatic-tests") - # Generate the payload for Deadline submission secondary_pool = ( self.deadline_pool_secondary or instance.data.get("secondaryPool") diff --git a/client/ayon_deadline/plugins/publish/submit_publish_job.py b/client/ayon_deadline/plugins/publish/submit_publish_job.py index 11bdc461f6..643dcc1c46 100644 --- a/client/ayon_deadline/plugins/publish/submit_publish_job.py +++ b/client/ayon_deadline/plugins/publish/submit_publish_job.py @@ -234,9 +234,6 @@ def _submit_deadline_post_job(self, instance, job, instances): "--targets", "farm" ] - if is_in_tests(): - args.append("--automatic-tests") - # Generate the payload for Deadline submission secondary_pool = ( self.deadline_pool_secondary or instance.data.get("secondaryPool") diff --git a/client/ayon_deadline/repository/custom/plugins/GlobalJobPreLoad.py b/client/ayon_deadline/repository/custom/plugins/GlobalJobPreLoad.py index 4081f02015..dbd1798608 100644 --- a/client/ayon_deadline/repository/custom/plugins/GlobalJobPreLoad.py +++ b/client/ayon_deadline/repository/custom/plugins/GlobalJobPreLoad.py @@ -502,8 +502,6 @@ def inject_ayon_environment(deadlinePlugin): "extractenvironments", export_url ] - if job.GetJobEnvironmentKeyValue("AYON_IN_TESTS"): - args.append("--automatic-tests") for key, value in add_kwargs.items(): args.extend(["--{}".format(key), value]) @@ -517,6 +515,10 @@ def inject_ayon_environment(deadlinePlugin): "AYON_API_KEY": ayon_api_key, "AYON_BUNDLE_NAME": ayon_bundle_name, } + + automatic_tests = job.GetJobEnvironmentKeyValue("AYON_IN_TESTS") + if automatic_tests: + environment["AYON_IN_TESTS"] = automatic_tests for env, val in environment.items(): # Add the env var for the Render Plugin that is about to render deadlinePlugin.SetEnvironmentVariable(env, val)