From 49c5b875a8a275adb6264474df31a2fc4be3584a Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 5 Dec 2024 16:19:58 +0100 Subject: [PATCH 1/8] Refactor removed filtering by frame Remainder has now real way to find frame pattern from single file. Doesn't make sense to filter on single file. --- client/ayon_core/pipeline/farm/pyblish_functions.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 559561c827..147763391b 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -435,13 +435,10 @@ def prepare_representations( " This may cause issues on farm." ).format(staging)) - files = _get_real_files_to_rendered( - [os.path.basename(remainder)], frames_to_render) - rep = { "name": ext, "ext": ext, - "files": files[0], + "files": os.path.basename(remainder), "stagingDir": staging, } From 2a20a9d169b2ed49eada6679166bd61a5db8889e Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 5 Dec 2024 16:20:57 +0100 Subject: [PATCH 2/8] Refactor filtering based on frame_to_render Previous implementation was naive and could be dangerous. Updated docstrings. Renamed. --- .../pipeline/farm/pyblish_functions.py | 47 ++++++++++++------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 147763391b..722dc4b5c6 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -385,8 +385,7 @@ def prepare_representations( if skeleton_data.get("slate"): frame_start -= 1 - files = _get_real_files_to_rendered(collection, frames_to_render) - + files = _get_real_files_to_render(collection, frames_to_render) # explicitly disable review by user preview = preview and not do_not_add_review rep = { @@ -492,31 +491,47 @@ def _get_real_frames_to_render(frames): return frames_to_render -def _get_real_files_to_rendered(collection, frames_to_render): - """Use expected files based on real frames_to_render. +def _get_real_files_to_render(collection, frames_to_render): + """Filter files with frames that should be really rendered. + + 'expected_files' are collected from DCC based on timeline setting. This is + being calculated differently in each DCC. Filtering here is on single place + + But artists might explicitly set frames they want to render in Publisher UI + This range would override and filter previously prepared expected files + from DCC. - Artists might explicitly set frames they want to render via Publisher UI. - This uses this value to filter out files Args: - frames_to_render (list): of str '1001' + collection (clique.Collection): absolute paths + frames_to_render (list[int]): of int 1001 + Returns: + (list[str]) + + Example: + -------- + + expectedFiles = [ + "foo_v01.0001.exr", + "foo_v01.0002.exr", + ] + frames_to_render = '0001' + >> + ["foo_v01.0001.exr"] - only explicitly requested frame returned """ - files = [os.path.basename(f) for f in list(collection)] - file_name, extracted_frame = list(collect_frames(files).items())[0] - - if not extracted_frame: - return files - - found_frame_pattern_length = len(extracted_frame) + found_frame_pattern_length = collection.padding normalized_frames_to_render = { str(frame_to_render).zfill(found_frame_pattern_length) for frame_to_render in frames_to_render } + head_name = os.path.basename(collection.head) + + file_names = [os.path.basename(f) for f in collection] return [ file_name - for file_name in files + for file_name in file_names if any( - frame in file_name + f"{head_name}{frame}{collection.tail}" == file_name for frame in normalized_frames_to_render ) ] From 81c71a757fae3ea0120b397017192336becb4a5a Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 5 Dec 2024 17:15:40 +0100 Subject: [PATCH 3/8] Prepare normalized expected file names Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- client/ayon_core/pipeline/farm/pyblish_functions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 722dc4b5c6..a7780ba97c 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -525,7 +525,10 @@ def _get_real_files_to_render(collection, frames_to_render): } head_name = os.path.basename(collection.head) - + normalized_filenames = { + f"{head_name}{frame}{collection.tail}" + for frame in normalized_frames_to_render + } file_names = [os.path.basename(f) for f in collection] return [ file_name From c886be22bdf2494e952c39d4f38959e35c732150 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 5 Dec 2024 17:19:19 +0100 Subject: [PATCH 4/8] Refactor logic to use set --- client/ayon_core/pipeline/farm/pyblish_functions.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index a7780ba97c..1f6e17972a 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -533,10 +533,7 @@ def _get_real_files_to_render(collection, frames_to_render): return [ file_name for file_name in file_names - if any( - f"{head_name}{frame}{collection.tail}" == file_name - for frame in normalized_frames_to_render - ) + if file_name in normalized_filenames ] From 06f6b519f0d45aa5e9dfadf3f184106d98166eb7 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 9 Dec 2024 12:42:07 +0100 Subject: [PATCH 5/8] Refactor logic to be even simpler --- .../pipeline/farm/pyblish_functions.py | 25 ++++++------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 1f6e17972a..902aa41af4 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -518,23 +518,14 @@ def _get_real_files_to_render(collection, frames_to_render): >> ["foo_v01.0001.exr"] - only explicitly requested frame returned """ - found_frame_pattern_length = collection.padding - normalized_frames_to_render = { - str(frame_to_render).zfill(found_frame_pattern_length) - for frame_to_render in frames_to_render - } - - head_name = os.path.basename(collection.head) - normalized_filenames = { - f"{head_name}{frame}{collection.tail}" - for frame in normalized_frames_to_render - } - file_names = [os.path.basename(f) for f in collection] - return [ - file_name - for file_name in file_names - if file_name in normalized_filenames - ] + included_frames = set(collection.indexes).intersection(frames_to_render) + real_collection = clique.Collection( + collection.head, + collection.tail, + collection.padding, + indexes=included_frames + ) + return list(real_collection) def create_instances_for_aov(instance, skeleton, aov_filter, From 6836a7f79b546e7dc500d2c8ba911469e3035012 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 9 Dec 2024 13:51:09 +0100 Subject: [PATCH 6/8] Fix docstring --- client/ayon_core/pipeline/farm/pyblish_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 902aa41af4..425b616adc 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -514,7 +514,7 @@ def _get_real_files_to_render(collection, frames_to_render): "foo_v01.0001.exr", "foo_v01.0002.exr", ] - frames_to_render = '0001' + frames_to_render = 1 >> ["foo_v01.0001.exr"] - only explicitly requested frame returned """ From d65865563f7aa34ede47b17e2dfc671a03f84255 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 9 Dec 2024 14:53:17 +0100 Subject: [PATCH 7/8] Fix expected files must be only file names --- client/ayon_core/pipeline/farm/pyblish_functions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 425b616adc..71068cb093 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -525,7 +525,8 @@ def _get_real_files_to_render(collection, frames_to_render): collection.padding, indexes=included_frames ) - return list(real_collection) + real_full_paths = list(real_collection) + return [os.path.basename(file_url) for file_url in real_full_paths] def create_instances_for_aov(instance, skeleton, aov_filter, From 4bf03f0a51ed5b73034b537f55d6c853fb572b3a Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 9 Dec 2024 15:39:08 +0100 Subject: [PATCH 8/8] Remove unused import 'collect_frames' from pyblish_functions --- client/ayon_core/pipeline/farm/pyblish_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 616e25596c..e48d99602e 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -7,7 +7,7 @@ from copy import deepcopy import attr import ayon_api import clique -from ayon_core.lib import Logger, collect_frames +from ayon_core.lib import Logger from ayon_core.pipeline import ( get_current_project_name, get_representation_path,