From d8c023d0a166b994d9ce0be1ec772b1247bbfffb Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 24 Feb 2025 15:14:32 +0100 Subject: [PATCH 01/27] Removed frame recalculation Maybe temporary commit as this naive approach won't be possible. Existing code recalculated/renamed rendered files according to values in AYON DB. This limits explicit render of selected only frames. --- client/ayon_core/plugins/publish/integrate.py | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/client/ayon_core/plugins/publish/integrate.py b/client/ayon_core/plugins/publish/integrate.py index e8fe09bab7..16122339f6 100644 --- a/client/ayon_core/plugins/publish/integrate.py +++ b/client/ayon_core/plugins/publish/integrate.py @@ -691,28 +691,6 @@ class IntegrateAsset(pyblish.api.InstancePlugin): # Use last frame for minimum padding # - that should cover both 'udim' and 'frame' minimum padding destination_padding = len(str(destination_indexes[-1])) - if not is_udim: - # Change padding for frames if template has defined higher - # padding. - template_padding = anatomy.templates_obj.frame_padding - if template_padding > destination_padding: - destination_padding = template_padding - - # If the representation has `frameStart` set it renumbers the - # frame indices of the published collection. It will start from - # that `frameStart` index instead. Thus if that frame start - # differs from the collection we want to shift the destination - # frame indices from the source collection. - # In case source are published in place we need to - # skip renumbering - repre_frame_start = repre.get("frameStart") - if repre_frame_start is not None: - index_frame_start = int(repre_frame_start) - # Shift destination sequence to the start frame - destination_indexes = [ - index_frame_start + idx - for idx in range(len(destination_indexes)) - ] # To construct the destination template with anatomy we require # a Frame or UDIM tile set for the template data. We use the first From 85c33580f1258b9c80ec09d5ec39a4c22ea7d193 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 26 Feb 2025 13:02:28 +0100 Subject: [PATCH 02/27] Revert "Removed frame recalculation" This reverts commit d8c023d0a166b994d9ce0be1ec772b1247bbfffb. --- client/ayon_core/plugins/publish/integrate.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/client/ayon_core/plugins/publish/integrate.py b/client/ayon_core/plugins/publish/integrate.py index 16122339f6..e8fe09bab7 100644 --- a/client/ayon_core/plugins/publish/integrate.py +++ b/client/ayon_core/plugins/publish/integrate.py @@ -691,6 +691,28 @@ class IntegrateAsset(pyblish.api.InstancePlugin): # Use last frame for minimum padding # - that should cover both 'udim' and 'frame' minimum padding destination_padding = len(str(destination_indexes[-1])) + if not is_udim: + # Change padding for frames if template has defined higher + # padding. + template_padding = anatomy.templates_obj.frame_padding + if template_padding > destination_padding: + destination_padding = template_padding + + # If the representation has `frameStart` set it renumbers the + # frame indices of the published collection. It will start from + # that `frameStart` index instead. Thus if that frame start + # differs from the collection we want to shift the destination + # frame indices from the source collection. + # In case source are published in place we need to + # skip renumbering + repre_frame_start = repre.get("frameStart") + if repre_frame_start is not None: + index_frame_start = int(repre_frame_start) + # Shift destination sequence to the start frame + destination_indexes = [ + index_frame_start + idx + for idx in range(len(destination_indexes)) + ] # To construct the destination template with anatomy we require # a Frame or UDIM tile set for the template data. We use the first From 080e86ee6eb731281901b69d2cd78fa709d27f21 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 26 Feb 2025 13:11:04 +0100 Subject: [PATCH 03/27] Do frame recalculation only for not explicit_frames `explicit_frames` denotes that artist wants to render only specific frames. Could be set in Publisher UI. --- client/ayon_core/plugins/publish/integrate.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/ayon_core/plugins/publish/integrate.py b/client/ayon_core/plugins/publish/integrate.py index e8fe09bab7..51d6333f27 100644 --- a/client/ayon_core/plugins/publish/integrate.py +++ b/client/ayon_core/plugins/publish/integrate.py @@ -706,7 +706,8 @@ class IntegrateAsset(pyblish.api.InstancePlugin): # In case source are published in place we need to # skip renumbering repre_frame_start = repre.get("frameStart") - if repre_frame_start is not None: + explicit_frames = instance.data.get("explicit_frames", False) + if not explicit_frames and repre_frame_start is not None: index_frame_start = int(repre_frame_start) # Shift destination sequence to the start frame destination_indexes = [ From 16e817f5d510034393ade00e96e4a45fd5dc7f48 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 26 Feb 2025 14:19:03 +0100 Subject: [PATCH 04/27] Fix propagation of explicit_frames --- client/ayon_core/pipeline/farm/pyblish_functions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index e48d99602e..c5a70dc022 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -247,7 +247,8 @@ def create_skeleton_instance( "useSequenceForReview": data.get("useSequenceForReview", True), # map inputVersions `ObjectId` -> `str` so json supports it "inputVersions": list(map(str, data.get("inputVersions", []))), - "colorspace": data.get("colorspace") + "colorspace": data.get("colorspace"), + "explicit_frames": data.get("explicit_frames") } if data.get("renderlayer"): From 594f1f1abb319434ede99e729be764ed2c55bd72 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 27 Feb 2025 11:56:11 +0100 Subject: [PATCH 05/27] Fix using explicit frames in Maya AOVs --- .../pipeline/farm/pyblish_functions.py | 60 ++++++++++++++----- 1 file changed, 44 insertions(+), 16 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index c5a70dc022..e9a9906d48 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -338,7 +338,7 @@ def prepare_representations( log = Logger.get_logger("farm_publishing") if frames_to_render is not None: - frames_to_render = _get_real_frames_to_render(frames_to_render) + frames_to_render = get_real_frames_to_render(frames_to_render) else: # Backwards compatibility for older logic frame_start = int(skeleton_data.get("frameStartHandle")) @@ -476,7 +476,7 @@ def prepare_representations( return representations -def _get_real_frames_to_render(frames): +def get_real_frames_to_render(frames): """Returns list of frames that should be rendered. Artists could want to selectively render only particular frames @@ -531,9 +531,14 @@ def _get_real_files_to_render(collection, frames_to_render): return [os.path.basename(file_url) for file_url in real_full_paths] -def create_instances_for_aov(instance, skeleton, aov_filter, - skip_integration_repre_list, - do_not_add_review): +def create_instances_for_aov( + instance, + skeleton, + aov_filter, + skip_integration_repre_list, + do_not_add_review, + frames_to_render=None +): """Create instances from AOVs. This will create new pyblish.api.Instances by going over expected @@ -591,7 +596,8 @@ def create_instances_for_aov(instance, skeleton, aov_filter, aov_filter, additional_color_data, skip_integration_repre_list, - do_not_add_review + do_not_add_review, + frames_to_render ) @@ -720,8 +726,15 @@ def get_product_name_and_group_from_template( return resulting_product_name, resulting_group_name -def _create_instances_for_aov(instance, skeleton, aov_filter, additional_data, - skip_integration_repre_list, do_not_add_review): +def _create_instances_for_aov( + instance, + skeleton, + aov_filter, + additional_data, + skip_integration_repre_list, + do_not_add_review, + frames_to_render=None +): """Create instance for each AOV found. This will create new instance for every AOV it can detect in expected @@ -735,7 +748,8 @@ def _create_instances_for_aov(instance, skeleton, aov_filter, additional_data, skip_integration_repre_list (list): list of extensions that shouldn't be published do_not_add_review (bool): explicitly disable review - + frames_to_render (str): implicit or explicit range of frames to render + this value is sent to Deadline in JobInfo.Frames Returns: list of instances @@ -755,10 +769,26 @@ def _create_instances_for_aov(instance, skeleton, aov_filter, additional_data, # go through AOVs in expected files for aov, files in expected_files[0].items(): collected_files = _collect_expected_files_for_aov(files) + staging_dir = ( + os.path.dirname(collected_files[0]) + if isinstance(collected_files, (list, tuple)) + else os.path.dirname(collected_files) + ) + + if frames_to_render is not None: + frames_to_render = get_real_frames_to_render(frames_to_render) + collections, _ = clique.assemble(collected_files) + collected_files = _get_real_files_to_render( + collections[0], frames_to_render) + else: + frame_start = int(skeleton.get("frameStartHandle")) + frame_end = int(skeleton.get("frameEndHandle")) + frames_to_render = list(range(frame_start, frame_end + 1)) - expected_filepath = collected_files if isinstance(collected_files, (list, tuple)): - expected_filepath = collected_files[0] + expected_filepath = os.path.join(staging_dir, collected_files[0]) + else: + expected_filepath = os.path.join(staging_dir, expected_filepath) dynamic_data = { "aov": aov, @@ -814,10 +844,8 @@ def _create_instances_for_aov(instance, skeleton, aov_filter, additional_data, dynamic_data=dynamic_data ) - staging = os.path.dirname(expected_filepath) - try: - staging = remap_source(staging, anatomy) + staging = remap_source(staging_dir, anatomy) except ValueError as e: log.warning(e) @@ -882,8 +910,8 @@ def _create_instances_for_aov(instance, skeleton, aov_filter, additional_data, "name": ext, "ext": ext, "files": collected_files, - "frameStart": int(skeleton["frameStartHandle"]), - "frameEnd": int(skeleton["frameEndHandle"]), + "frameStart": frames_to_render[0], + "frameEnd": frames_to_render[-1], # If expectedFile are absolute, we need only filenames "stagingDir": staging, "fps": new_instance.get("fps"), From 86a9613fdb849971f1f9fb2ddf16b3e28de3eca4 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 27 Feb 2025 13:06:45 +0100 Subject: [PATCH 06/27] Fix get real frames to include steps --- .../pipeline/farm/pyblish_functions.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index e9a9906d48..81e10abeff 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -480,13 +480,27 @@ def get_real_frames_to_render(frames): """Returns list of frames that should be rendered. Artists could want to selectively render only particular frames + Handles formats as: + 1001 + 1002,1004 + 1003-1005 + 1001-1100x5 """ + pattern = r'(?:x|step|by|every)?(\d+)$' + frames_to_render = [] + step = 1 for frame in frames.split(","): if "-" in frame: - splitted = frame.split("-") + frame_start, frame_end = frame.split("-") + match = re.findall(pattern, frame_end) + if match: + step = int(match[0]) + frame_end = re.sub(pattern, "", frame_end) + frames_to_render.extend( - range(int(splitted[0]), int(splitted[1])+1)) + range(int(frame_start), int(frame_end) + 1, step) + ) else: frames_to_render.append(int(frame)) frames_to_render.sort() From 8347bad343e0c1053b22472af7f29b478f4917b3 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 27 Feb 2025 13:08:25 +0100 Subject: [PATCH 07/27] Fix do frame recalculate only on lists --- client/ayon_core/pipeline/farm/pyblish_functions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 81e10abeff..e484232927 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -789,7 +789,10 @@ def _create_instances_for_aov( else os.path.dirname(collected_files) ) - if frames_to_render is not None: + if ( + frames_to_render is not None + and isinstance(collected_files, (list, tuple)) # not single file + ): frames_to_render = get_real_frames_to_render(frames_to_render) collections, _ = clique.assemble(collected_files) collected_files = _get_real_files_to_render( From 1455ebb54e4b7e6d20e2e059f78aa711387f9fe2 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 27 Feb 2025 13:08:43 +0100 Subject: [PATCH 08/27] Fix wrong variable --- client/ayon_core/pipeline/farm/pyblish_functions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index e484232927..c64bf91371 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -805,7 +805,8 @@ def _create_instances_for_aov( if isinstance(collected_files, (list, tuple)): expected_filepath = os.path.join(staging_dir, collected_files[0]) else: - expected_filepath = os.path.join(staging_dir, expected_filepath) + expected_filepath = os.path.join(staging_dir, collected_files) + dynamic_data = { "aov": aov, From 07674c304031ec60ebea391221145467d393d31e Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 27 Feb 2025 13:09:05 +0100 Subject: [PATCH 09/27] Fix against local variable issues --- client/ayon_core/pipeline/farm/pyblish_functions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index c64bf91371..f67eb1dd04 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -863,7 +863,7 @@ def _create_instances_for_aov( ) try: - staging = remap_source(staging_dir, anatomy) + staging_dir = remap_source(staging_dir, anatomy) except ValueError as e: log.warning(e) @@ -931,7 +931,7 @@ def _create_instances_for_aov( "frameStart": frames_to_render[0], "frameEnd": frames_to_render[-1], # If expectedFile are absolute, we need only filenames - "stagingDir": staging, + "stagingDir": staging_dir, "fps": new_instance.get("fps"), "tags": ["review"] if preview else [], "colorspaceData": { From ab3fdea704181a07c0f5da01c83cd7aa47b193ad Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 27 Feb 2025 13:17:02 +0100 Subject: [PATCH 10/27] Renamed new variable to hasExplicitFrames --- client/ayon_core/pipeline/farm/pyblish_functions.py | 2 +- client/ayon_core/plugins/publish/integrate.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index f67eb1dd04..5776e42ff6 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -248,7 +248,7 @@ def create_skeleton_instance( # map inputVersions `ObjectId` -> `str` so json supports it "inputVersions": list(map(str, data.get("inputVersions", []))), "colorspace": data.get("colorspace"), - "explicit_frames": data.get("explicit_frames") + "hasExplicitFrames": data.get("hasExplicitFrames") } if data.get("renderlayer"): diff --git a/client/ayon_core/plugins/publish/integrate.py b/client/ayon_core/plugins/publish/integrate.py index 51d6333f27..ae043a10a9 100644 --- a/client/ayon_core/plugins/publish/integrate.py +++ b/client/ayon_core/plugins/publish/integrate.py @@ -706,7 +706,7 @@ class IntegrateAsset(pyblish.api.InstancePlugin): # In case source are published in place we need to # skip renumbering repre_frame_start = repre.get("frameStart") - explicit_frames = instance.data.get("explicit_frames", False) + explicit_frames = instance.data.get("hasExplicitFrames", False) if not explicit_frames and repre_frame_start is not None: index_frame_start = int(repre_frame_start) # Shift destination sequence to the start frame From d150377092aec9997a33c2a27504b16902495113 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 28 Feb 2025 10:08:21 +0100 Subject: [PATCH 11/27] Fix gaps in color transcode Create sequence only if no gaps. --- client/ayon_core/plugins/publish/extract_color_transcode.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/client/ayon_core/plugins/publish/extract_color_transcode.py b/client/ayon_core/plugins/publish/extract_color_transcode.py index 3c11a016ec..22997dc986 100644 --- a/client/ayon_core/plugins/publish/extract_color_transcode.py +++ b/client/ayon_core/plugins/publish/extract_color_transcode.py @@ -280,6 +280,10 @@ class ExtractOIIOTranscode(publish.Extractor): collection = collections[0] frames = list(collection.indexes) + real_range = list(range(frames[0], frames[-1] + 1)) + if set(frames) != set(real_range): # check for gaps + return files_to_convert + frame_str = "{}-{}#".format(frames[0], frames[-1]) file_name = "{}{}{}".format(collection.head, frame_str, collection.tail) From 0bdc2bd29d80e0d49ed50bf06b2989f50604d770 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 28 Feb 2025 10:40:30 +0100 Subject: [PATCH 12/27] Fix pattern for occurence of : --- client/ayon_core/pipeline/farm/pyblish_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 5776e42ff6..c0285d0446 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -486,7 +486,7 @@ def get_real_frames_to_render(frames): 1003-1005 1001-1100x5 """ - pattern = r'(?:x|step|by|every)?(\d+)$' + pattern = r"(?:|step|by|every|:?)(\d+)$" frames_to_render = [] step = 1 From af2f210e7c08d15919c6efc75a1b66d424ad3f2e Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 28 Feb 2025 11:00:27 +0100 Subject: [PATCH 13/27] Fix pattern for occurence of : --- client/ayon_core/pipeline/farm/pyblish_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index c0285d0446..4795eaac47 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -486,7 +486,7 @@ def get_real_frames_to_render(frames): 1003-1005 1001-1100x5 """ - pattern = r"(?:|step|by|every|:?)(\d+)$" + pattern = r"(?:step|by|every|x|:)(\d+)$" frames_to_render = [] step = 1 From 380310b017d3e9df0c50cdbd4b416f02016db01a Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 10 Mar 2025 09:51:46 +0100 Subject: [PATCH 14/27] Removed optionality Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- client/ayon_core/pipeline/farm/pyblish_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 4795eaac47..80048ae0ee 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -747,7 +747,7 @@ def _create_instances_for_aov( additional_data, skip_integration_repre_list, do_not_add_review, - frames_to_render=None + frames_to_render ): """Create instance for each AOV found. From bafd3dcf8f787bbf694d02310811837f7cd589fc Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 10 Mar 2025 09:55:13 +0100 Subject: [PATCH 15/27] Update name of pattern --- client/ayon_core/pipeline/farm/pyblish_functions.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 80048ae0ee..037e6e4c42 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -486,17 +486,17 @@ def get_real_frames_to_render(frames): 1003-1005 1001-1100x5 """ - pattern = r"(?:step|by|every|x|:)(\d+)$" + step_pattern = re.compile(r"(?:step|by|every|x|:)(\d+)$") frames_to_render = [] step = 1 for frame in frames.split(","): if "-" in frame: frame_start, frame_end = frame.split("-") - match = re.findall(pattern, frame_end) + match = step_pattern.findall(frame_end) if match: step = int(match[0]) - frame_end = re.sub(pattern, "", frame_end) + frame_end = re.sub(step_pattern, "", frame_end) frames_to_render.extend( range(int(frame_start), int(frame_end) + 1, step) From 4ea2ed3dc1c18997013ab2b56f03ef2597288d93 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 10 Mar 2025 10:18:21 +0100 Subject: [PATCH 16/27] Updated use of first_filepath --- .../pipeline/farm/pyblish_functions.py | 26 +++++++------------ 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 037e6e4c42..b47b8eede0 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -393,7 +393,7 @@ def prepare_representations( rep = { "name": ext, "ext": ext, - "files": files, + "files": [os.path.basename(file_path) for file_path in files], "frameStart": frame_start, "frameEnd": frame_end, # If expectedFile are absolute, we need only filenames @@ -521,7 +521,7 @@ def _get_real_files_to_render(collection, frames_to_render): collection (clique.Collection): absolute paths frames_to_render (list[int]): of int 1001 Returns: - (list[str]) + (list[str]): absolut paths of files to be rendered Example: -------- @@ -541,8 +541,7 @@ def _get_real_files_to_render(collection, frames_to_render): collection.padding, indexes=included_frames ) - real_full_paths = list(real_collection) - return [os.path.basename(file_url) for file_url in real_full_paths] + return list(real_collection) def create_instances_for_aov( @@ -783,11 +782,10 @@ def _create_instances_for_aov( # go through AOVs in expected files for aov, files in expected_files[0].items(): collected_files = _collect_expected_files_for_aov(files) - staging_dir = ( - os.path.dirname(collected_files[0]) - if isinstance(collected_files, (list, tuple)) - else os.path.dirname(collected_files) - ) + first_filepath = collected_files + if isinstance(first_filepath, (list, tuple)): + first_filepath = first_filepath[0] + staging_dir = os.path.dirname(first_filepath) if ( frames_to_render is not None @@ -802,12 +800,6 @@ def _create_instances_for_aov( frame_end = int(skeleton.get("frameEndHandle")) frames_to_render = list(range(frame_start, frame_end + 1)) - if isinstance(collected_files, (list, tuple)): - expected_filepath = os.path.join(staging_dir, collected_files[0]) - else: - expected_filepath = os.path.join(staging_dir, collected_files) - - dynamic_data = { "aov": aov, "renderlayer": instance.data.get("renderlayer"), @@ -817,7 +809,7 @@ def _create_instances_for_aov( # TODO: this must be changed to be more robust. Any coincidence # of camera name in the file path will be considered as # camera name. This is not correct. - camera = [cam for cam in cameras if cam in expected_filepath] + camera = [cam for cam in cameras if cam in first_filepath] # Is there just one camera matching? # TODO: this is not true, we can have multiple cameras in the scene @@ -871,7 +863,7 @@ def _create_instances_for_aov( app = os.environ.get("AYON_HOST_NAME", "") - render_file_name = os.path.basename(expected_filepath) + render_file_name = os.path.basename(first_filepath) aov_patterns = aov_filter From 428175ee00051fc0eec0dbc00e4ac271c59c5d17 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Tue, 11 Mar 2025 11:14:13 +0100 Subject: [PATCH 17/27] rename '_get_real_files_to_render' to '_get_real_paths_to_render' --- client/ayon_core/pipeline/farm/pyblish_functions.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index b47b8eede0..202d84f02c 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -387,13 +387,18 @@ def prepare_representations( frame_start -= 1 frames_to_render.insert(0, frame_start) - files = _get_real_files_to_render(collection, frames_to_render) + filenames = [ + os.path.basename(filepath) + for filepath in _get_real_paths_to_render( + collection, frames_to_render + ) + ] # explicitly disable review by user preview = preview and not do_not_add_review rep = { "name": ext, "ext": ext, - "files": [os.path.basename(file_path) for file_path in files], + "files": filenames, "frameStart": frame_start, "frameEnd": frame_end, # If expectedFile are absolute, we need only filenames @@ -507,7 +512,7 @@ def get_real_frames_to_render(frames): return frames_to_render -def _get_real_files_to_render(collection, frames_to_render): +def _get_real_paths_to_render(collection, frames_to_render): """Filter files with frames that should be really rendered. 'expected_files' are collected from DCC based on timeline setting. This is @@ -793,7 +798,7 @@ def _create_instances_for_aov( ): frames_to_render = get_real_frames_to_render(frames_to_render) collections, _ = clique.assemble(collected_files) - collected_files = _get_real_files_to_render( + collected_files = _get_real_paths_to_render( collections[0], frames_to_render) else: frame_start = int(skeleton.get("frameStartHandle")) From 2f47a6d35ae5743c35d1a7b20b2bb58e5ce3df0e Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Tue, 11 Mar 2025 11:14:37 +0100 Subject: [PATCH 18/27] remove out of context comment --- client/ayon_core/pipeline/farm/pyblish_functions.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 202d84f02c..a9dcbcba70 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -399,10 +399,9 @@ def prepare_representations( "name": ext, "ext": ext, "files": filenames, + "stagingDir": staging, "frameStart": frame_start, "frameEnd": frame_end, - # If expectedFile are absolute, we need only filenames - "stagingDir": staging, "fps": skeleton_data.get("fps"), "tags": ["review"] if preview else [], } From 1255aa2776f6ed4a8956995d285bd28793129107 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Tue, 11 Mar 2025 11:23:01 +0100 Subject: [PATCH 19/27] added typehints and better examples --- .../pipeline/farm/pyblish_functions.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index a9dcbcba70..36a231d45f 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -1,3 +1,4 @@ +from __future__ import annotations import copy import os import re @@ -480,15 +481,23 @@ def prepare_representations( return representations -def get_real_frames_to_render(frames): +def get_real_frames_to_render(frames: str) -> list[int]: """Returns list of frames that should be rendered. Artists could want to selectively render only particular frames + Handles formats as: - 1001 - 1002,1004 - 1003-1005 - 1001-1100x5 + - '1001' > [1001] + - '1002,1004' > [1002, 1004] + - '1003-1005' > [1003, 1004, 1005] + - '1001-1021x5' > [1001, 1006, 1011, 1016, 2021] + + Args: + frames (str): string with frames to render + + Returns: + list[int]: List of frames. + """ step_pattern = re.compile(r"(?:step|by|every|x|:)(\d+)$") From 369383045f6dc2e48b187d720d874dd512730e62 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Tue, 11 Mar 2025 11:47:47 +0100 Subject: [PATCH 20/27] rename 'get_real_frames_to_render' to 'convert_frames_str_to_list' --- .../pipeline/farm/pyblish_functions.py | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 36a231d45f..cbc44e725e 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -339,7 +339,7 @@ def prepare_representations( log = Logger.get_logger("farm_publishing") if frames_to_render is not None: - frames_to_render = get_real_frames_to_render(frames_to_render) + frames_to_render = convert_frames_str_to_list(frames_to_render) else: # Backwards compatibility for older logic frame_start = int(skeleton_data.get("frameStartHandle")) @@ -481,19 +481,21 @@ def prepare_representations( return representations -def get_real_frames_to_render(frames: str) -> list[int]: - """Returns list of frames that should be rendered. - - Artists could want to selectively render only particular frames +def convert_frames_str_to_list(frames: str) -> list[int]: + """Convert frames definition string to frames. Handles formats as: - - '1001' > [1001] - - '1002,1004' > [1002, 1004] - - '1003-1005' > [1003, 1004, 1005] - - '1001-1021x5' > [1001, 1006, 1011, 1016, 2021] + >>> convert_frames_str_to_list('1001') + [1001] + >>> convert_frames_str_to_list('1002,1004') + [1002, 1004] + >>> convert_frames_str_to_list('1003-1005') + [1003, 1004, 1005] + >>> convert_frames_str_to_list('1001-1021x5') + [1001, 1006, 1011, 1016, 1021] Args: - frames (str): string with frames to render + frames (str): String with frames definition. Returns: list[int]: List of frames. @@ -501,7 +503,7 @@ def get_real_frames_to_render(frames: str) -> list[int]: """ step_pattern = re.compile(r"(?:step|by|every|x|:)(\d+)$") - frames_to_render = [] + output = [] step = 1 for frame in frames.split(","): if "-" in frame: @@ -511,13 +513,13 @@ def get_real_frames_to_render(frames: str) -> list[int]: step = int(match[0]) frame_end = re.sub(step_pattern, "", frame_end) - frames_to_render.extend( + output.extend( range(int(frame_start), int(frame_end) + 1, step) ) else: - frames_to_render.append(int(frame)) - frames_to_render.sort() - return frames_to_render + output.append(int(frame)) + output.sort() + return output def _get_real_paths_to_render(collection, frames_to_render): @@ -804,7 +806,7 @@ def _create_instances_for_aov( frames_to_render is not None and isinstance(collected_files, (list, tuple)) # not single file ): - frames_to_render = get_real_frames_to_render(frames_to_render) + frames_to_render = convert_frames_str_to_list(frames_to_render) collections, _ = clique.assemble(collected_files) collected_files = _get_real_paths_to_render( collections[0], frames_to_render) From 5a88698ce14bf87a6feca460724391effeab6600 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Tue, 11 Mar 2025 12:10:02 +0100 Subject: [PATCH 21/27] rename '_get_real_paths_to_render' back to '_get_real_files_to_render' --- client/ayon_core/pipeline/farm/pyblish_functions.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index cbc44e725e..30ca506879 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -390,7 +390,7 @@ def prepare_representations( filenames = [ os.path.basename(filepath) - for filepath in _get_real_paths_to_render( + for filepath in _get_real_files_to_render( collection, frames_to_render ) ] @@ -522,7 +522,7 @@ def convert_frames_str_to_list(frames: str) -> list[int]: return output -def _get_real_paths_to_render(collection, frames_to_render): +def _get_real_files_to_render(collection, frames_to_render): """Filter files with frames that should be really rendered. 'expected_files' are collected from DCC based on timeline setting. This is @@ -808,7 +808,7 @@ def _create_instances_for_aov( ): frames_to_render = convert_frames_str_to_list(frames_to_render) collections, _ = clique.assemble(collected_files) - collected_files = _get_real_paths_to_render( + collected_files = _get_real_files_to_render( collections[0], frames_to_render) else: frame_start = int(skeleton.get("frameStartHandle")) From 3853fab3f432c97cc0d8abd51fdd2cd03f57e10d Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Tue, 11 Mar 2025 12:10:28 +0100 Subject: [PATCH 22/27] fix return description --- client/ayon_core/pipeline/farm/pyblish_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 30ca506879..407bab43b9 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -536,7 +536,7 @@ def _get_real_files_to_render(collection, frames_to_render): collection (clique.Collection): absolute paths frames_to_render (list[int]): of int 1001 Returns: - (list[str]): absolut paths of files to be rendered + list[str]: absolute paths of files to be rendered Example: -------- From 44ba00e22a3ab344a904d8164353e542335bace3 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Tue, 11 Mar 2025 12:10:35 +0100 Subject: [PATCH 23/27] better example --- .../pipeline/farm/pyblish_functions.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 407bab43b9..39b4bd528e 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -532,22 +532,23 @@ def _get_real_files_to_render(collection, frames_to_render): This range would override and filter previously prepared expected files from DCC. + Example: + >>> expected_files = clique.parse([ + >>> "foo_v01.0001.exr", + >>> "foo_v01.0002.exr", + >>> ]) + >>> frames_to_render = [1] + >>> _get_real_files_to_render(expected_files, frames_to_render) + ["foo_v01.0001.exr"] + Args: collection (clique.Collection): absolute paths frames_to_render (list[int]): of int 1001 + Returns: list[str]: absolute paths of files to be rendered - Example: - -------- - expectedFiles = [ - "foo_v01.0001.exr", - "foo_v01.0002.exr", - ] - frames_to_render = 1 - >> - ["foo_v01.0001.exr"] - only explicitly requested frame returned """ included_frames = set(collection.indexes).intersection(frames_to_render) real_collection = clique.Collection( From 74cb8574eedc9a8947f00048c14a402571bd1ba8 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Tue, 11 Mar 2025 12:11:32 +0100 Subject: [PATCH 24/27] fix typehint for 'frames_to_render' --- client/ayon_core/pipeline/farm/pyblish_functions.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 39b4bd528e..0261a0c2b5 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -326,8 +326,8 @@ def prepare_representations( skip_integration_repre_list (list): exclude specific extensions, do_not_add_review (bool): explicitly skip review color_managed_plugin (publish.ColormanagedPyblishPluginMixin) - frames_to_render (str): implicit or explicit range of frames to render - this value is sent to Deadline in JobInfo.Frames + frames_to_render (str | None): implicit or explicit range of frames + to render this value is sent to Deadline in JobInfo.Frames Returns: list of representations @@ -579,6 +579,7 @@ def create_instances_for_aov( aov_filter (dict): AOV filter. skip_integration_repre_list (list): skip do_not_add_review (bool): Explicitly disable reviews + frames_to_render (str | None): Frames to render. Returns: list of pyblish.api.Instance: Instances created from @@ -777,8 +778,8 @@ def _create_instances_for_aov( skip_integration_repre_list (list): list of extensions that shouldn't be published do_not_add_review (bool): explicitly disable review - frames_to_render (str): implicit or explicit range of frames to render - this value is sent to Deadline in JobInfo.Frames + frames_to_render (str | None): implicit or explicit range of + frames to render this value is sent to Deadline in JobInfo.Frames Returns: list of instances From d473118d392c4f78618ceaece50538251870d7dd Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 11 Mar 2025 18:19:34 +0100 Subject: [PATCH 25/27] Used clique method Co-authored-by: Roy Nieterau --- client/ayon_core/plugins/publish/extract_color_transcode.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/client/ayon_core/plugins/publish/extract_color_transcode.py b/client/ayon_core/plugins/publish/extract_color_transcode.py index 22997dc986..1f2c2a89af 100644 --- a/client/ayon_core/plugins/publish/extract_color_transcode.py +++ b/client/ayon_core/plugins/publish/extract_color_transcode.py @@ -280,8 +280,7 @@ class ExtractOIIOTranscode(publish.Extractor): collection = collections[0] frames = list(collection.indexes) - real_range = list(range(frames[0], frames[-1] + 1)) - if set(frames) != set(real_range): # check for gaps + if collection.holes(): return files_to_convert frame_str = "{}-{}#".format(frames[0], frames[-1]) From 58e77209b3556459826d19ae9288b31cd167eeac Mon Sep 17 00:00:00 2001 From: Ynbot Date: Tue, 11 Mar 2025 17:50:15 +0000 Subject: [PATCH 26/27] [Automated] Add generated package files from main --- client/ayon_core/version.py | 2 +- package.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/client/ayon_core/version.py b/client/ayon_core/version.py index e533e08fe4..1157c53d56 100644 --- a/client/ayon_core/version.py +++ b/client/ayon_core/version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- """Package declaring AYON addon 'core' version.""" -__version__ = "1.1.3+dev" +__version__ = "1.1.4" diff --git a/package.py b/package.py index 02e2f25384..0ad98648d9 100644 --- a/package.py +++ b/package.py @@ -1,6 +1,6 @@ name = "core" title = "Core" -version = "1.1.3+dev" +version = "1.1.4" client_dir = "ayon_core" diff --git a/pyproject.toml b/pyproject.toml index f065ca0c39..a16d0ef4a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ [tool.poetry] name = "ayon-core" -version = "1.1.3+dev" +version = "1.1.4" description = "" authors = ["Ynput Team "] readme = "README.md" From fbdc70f396cfebc4f1ed6a540e8d85cbeb0b5a56 Mon Sep 17 00:00:00 2001 From: Ynbot Date: Tue, 11 Mar 2025 17:50:55 +0000 Subject: [PATCH 27/27] [Automated] Update version in package.py for develop --- client/ayon_core/version.py | 2 +- package.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/client/ayon_core/version.py b/client/ayon_core/version.py index 1157c53d56..332001aef7 100644 --- a/client/ayon_core/version.py +++ b/client/ayon_core/version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- """Package declaring AYON addon 'core' version.""" -__version__ = "1.1.4" +__version__ = "1.1.4+dev" diff --git a/package.py b/package.py index 0ad98648d9..2c9072d443 100644 --- a/package.py +++ b/package.py @@ -1,6 +1,6 @@ name = "core" title = "Core" -version = "1.1.4" +version = "1.1.4+dev" client_dir = "ayon_core" diff --git a/pyproject.toml b/pyproject.toml index a16d0ef4a3..0bbe0f90e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ [tool.poetry] name = "ayon-core" -version = "1.1.4" +version = "1.1.4+dev" description = "" authors = ["Ynput Team "] readme = "README.md"