From 2f6ca5a2385bd3073961cf49bca220e9d3fb88b9 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 18 Oct 2024 18:35:27 +0200 Subject: [PATCH 01/12] Implemented explicit frames for simple files representations --- .../pipeline/farm/pyblish_functions.py | 79 ++++++++++++++++--- 1 file changed, 69 insertions(+), 10 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 98951b2766..5908644dca 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -7,7 +7,7 @@ import attr import ayon_api import clique -from ayon_core.lib import Logger +from ayon_core.lib import Logger, collect_frames from ayon_core.pipeline import get_current_project_name, get_representation_path from ayon_core.pipeline.create import get_product_name from ayon_core.pipeline.farm.patterning import match_aov_pattern @@ -295,11 +295,17 @@ def _add_review_families(families): return families -def prepare_representations(skeleton_data, exp_files, anatomy, aov_filter, - skip_integration_repre_list, - do_not_add_review, - context, - color_managed_plugin): +def prepare_representations( + skeleton_data, + exp_files, + anatomy, + aov_filter, + skip_integration_repre_list, + do_not_add_review, + context, + color_managed_plugin, + frames_to_render +): """Create representations for file sequences. This will return representations of expected files if they are not @@ -315,6 +321,8 @@ def prepare_representations(skeleton_data, exp_files, anatomy, aov_filter, skip_integration_repre_list (list): exclude specific extensions, do_not_add_review (bool): explicitly skip review color_managed_plugin (publish.ColormanagedPyblishPluginMixin) + frames_to_render (str): implicit or explicit range of frames to render + this value is sent to Deadline in JobInfo.Frames Returns: list of representations @@ -325,6 +333,8 @@ def prepare_representations(skeleton_data, exp_files, anatomy, aov_filter, log = Logger.get_logger("farm_publishing") + frames_to_render = _get_real_frames_to_render(frames_to_render) + # create representation for every collected sequence for collection in collections: ext = collection.tail.lstrip(".") @@ -361,18 +371,21 @@ def prepare_representations(skeleton_data, exp_files, anatomy, aov_filter, " This may cause issues on farm." ).format(staging)) - frame_start = int(skeleton_data.get("frameStartHandle")) + frame_start = int(frames_to_render[0]) + frame_end = int(frames_to_render[-1]) if skeleton_data.get("slate"): frame_start -= 1 + files = _get_real_files_to_rendered(collection, frames_to_render) + # explicitly disable review by user preview = preview and not do_not_add_review rep = { "name": ext, "ext": ext, - "files": [os.path.basename(f) for f in list(collection)], + "files": files, "frameStart": frame_start, - "frameEnd": int(skeleton_data.get("frameEndHandle")), + "frameEnd": frame_end, # If expectedFile are absolute, we need only filenames "stagingDir": staging, "fps": skeleton_data.get("fps"), @@ -413,10 +426,13 @@ def prepare_representations(skeleton_data, exp_files, anatomy, aov_filter, " This may cause issues on farm." ).format(staging)) + files = _get_real_files_to_rendered( + [os.path.basename(remainder)], frames_to_render) + rep = { "name": ext, "ext": ext, - "files": os.path.basename(remainder), + "files": files[0], "stagingDir": staging, } @@ -453,6 +469,49 @@ def prepare_representations(skeleton_data, exp_files, anatomy, aov_filter, return representations +def _get_real_frames_to_render(frames): + """Returns list of frames that should be rendered. + + Artists could want to selectively render only particular frames + """ + frames_to_render = [] + for frame in frames.split(","): + if "-" in frame: + splitted = frame.split("-") + frames_to_render.extend(range(int(splitted[0]), int(splitted[1]))) + else: + frames_to_render.append(frame) + return [str(frame_to_render) for frame_to_render in frames_to_render] + + +def _get_real_files_to_rendered(collection, frames_to_render): + """Use expected files based on real frames_to_render. + + Artists might explicitly set frames they want to render via Publisher UI. + This uses this value to filter out files + Args: + frames_to_render (list): of str '1001' + """ + files = [os.path.basename(f) for f in list(collection)] + file_name, extracted_frame = list(collect_frames(files).items())[0] + if extracted_frame: + found_frame_pattern_length = len(extracted_frame) + normalized_frames_to_render = set() + for frame_to_render in frames_to_render: + normalized_frames_to_render.add( + str(frame_to_render).zfill(found_frame_pattern_length) + ) + + filtered_files = [] + for file_name in files: + if any(frame in file_name + for frame in normalized_frames_to_render): + filtered_files.append(file_name) + + files = filtered_files + return files + + def create_instances_for_aov(instance, skeleton, aov_filter, skip_integration_repre_list, do_not_add_review): From 8a074daa2bb226de239d6d8291069c8796398086 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 4 Nov 2024 16:56:41 +0100 Subject: [PATCH 02/12] Fix single frame render --- client/ayon_core/pipeline/farm/pyblish_functions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 5908644dca..c70967dfc1 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -478,7 +478,8 @@ def _get_real_frames_to_render(frames): for frame in frames.split(","): if "-" in frame: splitted = frame.split("-") - frames_to_render.extend(range(int(splitted[0]), int(splitted[1]))) + frames_to_render.extend( + range(int(splitted[0]), int(splitted[1])+1)) else: frames_to_render.append(frame) return [str(frame_to_render) for frame_to_render in frames_to_render] From 87bb613b751ea508ae54a9813bf6eb5852ca5b6b Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 7 Nov 2024 17:38:00 +0100 Subject: [PATCH 03/12] Added optionality to new argument in method signature --- client/ayon_core/pipeline/farm/pyblish_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index c70967dfc1..e9f179c668 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -304,7 +304,7 @@ def prepare_representations( do_not_add_review, context, color_managed_plugin, - frames_to_render + frames_to_render=None ): """Create representations for file sequences. From c56cd07e67b22b4e34cf6c246229150799fa5ab0 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 21 Nov 2024 12:47:37 +0100 Subject: [PATCH 04/12] Provided backward compatibility for prepare_representations --- client/ayon_core/pipeline/farm/pyblish_functions.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index e9f179c668..e236ec6c3d 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -333,7 +333,13 @@ def prepare_representations( log = Logger.get_logger("farm_publishing") - frames_to_render = _get_real_frames_to_render(frames_to_render) + if frames_to_render is not None: + frames_to_render = _get_real_frames_to_render(frames_to_render) + else: + # Backwards compatibility for older logic + frame_start = int(skeleton_data.get("frameStartHandle")) + frame_end = int(skeleton_data.get("frameEndHandle")) + frames_to_render = list(range(frame_start, frame_end + 1)) # create representation for every collected sequence for collection in collections: From c2716872d43d20f1e7de051375244893cd192d38 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 21 Nov 2024 12:49:24 +0100 Subject: [PATCH 05/12] Do not convert to str unnecessary Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- client/ayon_core/pipeline/farm/pyblish_functions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index e9f179c668..aa69633a22 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -482,7 +482,8 @@ def _get_real_frames_to_render(frames): range(int(splitted[0]), int(splitted[1])+1)) else: frames_to_render.append(frame) - return [str(frame_to_render) for frame_to_render in frames_to_render] + frames_to_render.sort() + return frames_to_render def _get_real_files_to_rendered(collection, frames_to_render): From de88260ddac22419b3a87606aaae08bfc9ec4e09 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 21 Nov 2024 12:50:23 +0100 Subject: [PATCH 06/12] frames_to_render are now list of integers Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- client/ayon_core/pipeline/farm/pyblish_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index aa69633a22..876a5b504f 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -481,7 +481,7 @@ def _get_real_frames_to_render(frames): frames_to_render.extend( range(int(splitted[0]), int(splitted[1])+1)) else: - frames_to_render.append(frame) + frames_to_render.append(int(frame)) frames_to_render.sort() return frames_to_render From 630d2d49130a9cea142aea999203fc00106a269d Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 21 Nov 2024 12:50:36 +0100 Subject: [PATCH 07/12] frames_to_render are now list of integers Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- client/ayon_core/pipeline/farm/pyblish_functions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 876a5b504f..6740950d78 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -371,8 +371,8 @@ def prepare_representations( " This may cause issues on farm." ).format(staging)) - frame_start = int(frames_to_render[0]) - frame_end = int(frames_to_render[-1]) + frame_start = frames_to_render[0] + frame_end = frames_to_render[-1] if skeleton_data.get("slate"): frame_start -= 1 From 5f3175258725853011ab4b9eb2c58c1fc6959eda Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 21 Nov 2024 12:50:59 +0100 Subject: [PATCH 08/12] Used comprehension Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- client/ayon_core/pipeline/farm/pyblish_functions.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 6740950d78..09df371371 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -498,11 +498,10 @@ def _get_real_files_to_rendered(collection, frames_to_render): file_name, extracted_frame = list(collect_frames(files).items())[0] if extracted_frame: found_frame_pattern_length = len(extracted_frame) - normalized_frames_to_render = set() - for frame_to_render in frames_to_render: - normalized_frames_to_render.add( - str(frame_to_render).zfill(found_frame_pattern_length) - ) + normalized_frames_to_render = { + str(frame_to_render).zfill(found_frame_pattern_length) + for frame_to_render in frames_to_render + } filtered_files = [] for file_name in files: From fd20885ac26366b996c2b14d102cbefdc3a75341 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 21 Nov 2024 12:51:12 +0100 Subject: [PATCH 09/12] Used comprehension Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- .../ayon_core/pipeline/farm/pyblish_functions.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 09df371371..4ba088f92c 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -503,13 +503,14 @@ def _get_real_files_to_rendered(collection, frames_to_render): for frame_to_render in frames_to_render } - filtered_files = [] - for file_name in files: - if any(frame in file_name - for frame in normalized_frames_to_render): - filtered_files.append(file_name) - - files = filtered_files + files = [ + filename + for filename in files + if any( + frame in filename + for frame in normalized_frames_to_render + ) + ] return files From 80ab628d44b083f168e1a7e822f4f1145d40145b Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 21 Nov 2024 12:52:51 +0100 Subject: [PATCH 10/12] Changed condition to bail early --- .../pipeline/farm/pyblish_functions.py | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index e236ec6c3d..37a018e116 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -501,21 +501,23 @@ def _get_real_files_to_rendered(collection, frames_to_render): """ files = [os.path.basename(f) for f in list(collection)] file_name, extracted_frame = list(collect_frames(files).items())[0] - if extracted_frame: - found_frame_pattern_length = len(extracted_frame) - normalized_frames_to_render = set() - for frame_to_render in frames_to_render: - normalized_frames_to_render.add( - str(frame_to_render).zfill(found_frame_pattern_length) - ) + if not extracted_frame: + return files + + found_frame_pattern_length = len(extracted_frame) + normalized_frames_to_render = set() + for frame_to_render in frames_to_render: + normalized_frames_to_render.add( + str(frame_to_render).zfill(found_frame_pattern_length) + ) - filtered_files = [] - for file_name in files: - if any(frame in file_name - for frame in normalized_frames_to_render): - filtered_files.append(file_name) + filtered_files = [] + for file_name in files: + if any(frame in file_name + for frame in normalized_frames_to_render): + filtered_files.append(file_name) - files = filtered_files + files = filtered_files return files From 63592f9e2bb799350338e5f29e39cdc59bd28077 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 21 Nov 2024 14:32:54 +0100 Subject: [PATCH 11/12] Used comprehension Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- client/ayon_core/pipeline/farm/pyblish_functions.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index e1d83a175e..e3470f4c41 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -507,11 +507,10 @@ def _get_real_files_to_rendered(collection, frames_to_render): return files found_frame_pattern_length = len(extracted_frame) - normalized_frames_to_render = set() - for frame_to_render in frames_to_render: - normalized_frames_to_render.add( - str(frame_to_render).zfill(found_frame_pattern_length) - ) + normalized_frames_to_render = { + str(frame_to_render).zfill(found_frame_pattern_length) + for frame_to_render in frames_to_render + } filtered_files = [] for file_name in files: From 112c4bdc0eec32f7d140964a911d88c5926a27e7 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 21 Nov 2024 14:33:08 +0100 Subject: [PATCH 12/12] Used comprehension Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- .../ayon_core/pipeline/farm/pyblish_functions.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index e3470f4c41..16364a17ee 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -512,14 +512,14 @@ def _get_real_files_to_rendered(collection, frames_to_render): for frame_to_render in frames_to_render } - filtered_files = [] - for file_name in files: - if any(frame in file_name - for frame in normalized_frames_to_render): - filtered_files.append(file_name) - - files = filtered_files - return files + return [ + file_name + for file_name in files + if any( + frame in file_name + for frame in normalized_frames_to_render + ) + ] def create_instances_for_aov(instance, skeleton, aov_filter,