From f9a0b7a1ec62f42d9a96ed2524671e792195849f Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Thu, 11 Apr 2024 12:58:47 +0200 Subject: [PATCH 01/16] Replace instances while preserving their `instance.id` and instance.data[`instance_id`] --- .../plugins/publish/collect_render.py | 11 ++---- .../fusion/plugins/publish/collect_render.py | 16 +++----- .../publish/abstract_collect_render.py | 15 +++++++- client/ayon_core/pipeline/publish/lib.py | 38 +++++++++++++++++++ 4 files changed, 60 insertions(+), 20 deletions(-) diff --git a/client/ayon_core/hosts/aftereffects/plugins/publish/collect_render.py b/client/ayon_core/hosts/aftereffects/plugins/publish/collect_render.py index 4134e9d593..46616a2d93 100644 --- a/client/ayon_core/hosts/aftereffects/plugins/publish/collect_render.py +++ b/client/ayon_core/hosts/aftereffects/plugins/publish/collect_render.py @@ -41,7 +41,6 @@ class CollectAERender(publish.AbstractCollectRender): def get_instances(self, context): instances = [] - instances_to_remove = [] app_version = CollectAERender.get_stub().get_app_version() app_version = app_version[0:4] @@ -117,7 +116,10 @@ class CollectAERender(publish.AbstractCollectRender): fps=fps, app_version=app_version, publish_attributes=inst.data.get("publish_attributes", {}), - file_names=[item.file_name for item in render_q] + file_names=[item.file_name for item in render_q], + + # The source instance this render instance replaces + source_instance=inst ) comp = compositions_by_id.get(comp_id) @@ -144,11 +146,6 @@ class CollectAERender(publish.AbstractCollectRender): # to skip ExtractReview locally instance.families.remove("review") - instances.append(instance) - instances_to_remove.append(inst) - - for instance in instances_to_remove: - context.remove(instance) return instances def get_expected_files(self, render_instance): diff --git a/client/ayon_core/hosts/fusion/plugins/publish/collect_render.py b/client/ayon_core/hosts/fusion/plugins/publish/collect_render.py index 36102d02cb..521a47675a 100644 --- a/client/ayon_core/hosts/fusion/plugins/publish/collect_render.py +++ b/client/ayon_core/hosts/fusion/plugins/publish/collect_render.py @@ -37,14 +37,13 @@ class CollectFusionRender( aspect_x = comp_frame_format_prefs["AspectX"] aspect_y = comp_frame_format_prefs["AspectY"] - instances = [] - instances_to_remove = [] current_file = context.data["currentFile"] version = context.data["version"] project_entity = context.data["projectEntity"] + instances = [] for inst in context: if not inst.data.get("active", True): continue @@ -91,7 +90,10 @@ class CollectFusionRender( frameStep=1, fps=comp_frame_format_prefs.get("Rate"), app_version=comp.GetApp().Version, - publish_attributes=inst.data.get("publish_attributes", {}) + publish_attributes=inst.data.get("publish_attributes", {}), + + # The source instance this render instance replaces + source_instance=inst ) render_target = inst.data["creator_attributes"]["render_target"] @@ -114,14 +116,6 @@ class CollectFusionRender( # to skip ExtractReview locally instance.families.remove("review") - # add new instance to the list and remove the original - # instance since it is not needed anymore - instances.append(instance) - instances_to_remove.append(inst) - - for instance in instances_to_remove: - context.remove(instance) - return instances def post_collecting_action(self): diff --git a/client/ayon_core/pipeline/publish/abstract_collect_render.py b/client/ayon_core/pipeline/publish/abstract_collect_render.py index 745632ca0a..1b482b1d5f 100644 --- a/client/ayon_core/pipeline/publish/abstract_collect_render.py +++ b/client/ayon_core/pipeline/publish/abstract_collect_render.py @@ -12,6 +12,7 @@ import six import pyblish.api from .publish_plugins import AbstractMetaContextPlugin +from .lib import replace_instance_in_context @attr.s @@ -81,6 +82,9 @@ class RenderInstance(object): outputDir = attr.ib(default=None) context = attr.ib(default=None) + # The source instance this render instance should replace in the context + source_instance = attr.ib(default=None, type=pyblish.api.Instance) + @frameStart.validator def check_frame_start(self, _, value): """Validate if frame start is not larger then end.""" @@ -214,8 +218,15 @@ class AbstractCollectRender(pyblish.api.ContextPlugin): data = self.add_additional_data(data) render_instance_dict = attr.asdict(render_instance) - instance = context.create_instance(render_instance.name) - instance.data["label"] = render_instance.label + instance = pyblish.api.Instance(render_instance.name, + parent=context) + if render_instance.source_instance: + replace_instance_in_context(context, + render_instance.source_instance, + instance) + else: + context.append(instance) + instance.data.update(render_instance_dict) instance.data.update(data) diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index 8d3644637b..63fb0feb0a 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -934,3 +934,41 @@ def get_publish_instance_families(instance): families.discard(family) output.extend(families) return output + + +def replace_instance_in_context( + context: pyblish.api.Context, + source_instance: pyblish.api.Instance, + destination_instance: pyblish.api.Instance +): + """Replace source instance with the destination instance. + + This transfers the instance's IDs so that the new instance acts exactly + as if it was the source instance the whole time. This is required for + the publisher to correctly detect and transfer the logs relevant for the + instance. + + Returns: + bool: Whether the source instance was replaced or the destination + instance was only added because source instance did not exist + in the context. + + """ + + # Transfer the pyblish.api.Instance id + destination_instance._id = source_instance.id + + # Transfer the `instance_id` of the new publisher's instances + key = "instance_id" + if key in source_instance.data: + destination_instance.data[key] = source_instance.data[key] + + # Replace the instance at the same index in the context + for i, instance in enumerate(context): + if source_instance is instance: + context[i] = destination_instance + return True + + # Source was not found + context.append(destination_instance) + return False From 56fd07a7848a4cb0c3ee5d16d60d84e39c4333e5 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Thu, 11 Apr 2024 13:14:24 +0200 Subject: [PATCH 02/16] Fix replacement --- client/ayon_core/pipeline/publish/abstract_collect_render.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/publish/abstract_collect_render.py b/client/ayon_core/pipeline/publish/abstract_collect_render.py index 1b482b1d5f..01c2ed7537 100644 --- a/client/ayon_core/pipeline/publish/abstract_collect_render.py +++ b/client/ayon_core/pipeline/publish/abstract_collect_render.py @@ -220,7 +220,7 @@ class AbstractCollectRender(pyblish.api.ContextPlugin): instance = pyblish.api.Instance(render_instance.name, parent=context) - if render_instance.source_instance: + if render_instance.source_instance is not None: replace_instance_in_context(context, render_instance.source_instance, instance) From 6e812e00796c8f0a2cf3eedf5d5d00196d3984d2 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Thu, 11 Apr 2024 13:23:00 +0200 Subject: [PATCH 03/16] Fix replacement, and do not add new instance in context twice --- .../pipeline/publish/abstract_collect_render.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/client/ayon_core/pipeline/publish/abstract_collect_render.py b/client/ayon_core/pipeline/publish/abstract_collect_render.py index 01c2ed7537..096f1c406e 100644 --- a/client/ayon_core/pipeline/publish/abstract_collect_render.py +++ b/client/ayon_core/pipeline/publish/abstract_collect_render.py @@ -218,14 +218,15 @@ class AbstractCollectRender(pyblish.api.ContextPlugin): data = self.add_additional_data(data) render_instance_dict = attr.asdict(render_instance) - instance = pyblish.api.Instance(render_instance.name, - parent=context) + instance = context.create_instance(render_instance.name) if render_instance.source_instance is not None: - replace_instance_in_context(context, - render_instance.source_instance, - instance) - else: - context.append(instance) + # remove the new instance, because we want to insert it + # at the position of the original instance + context.pop() + replace_instance_in_context( + context, + source_instance=render_instance.source_instance, + destination_instance=instance) instance.data.update(render_instance_dict) instance.data.update(data) From 5b85cc9ec3cafbe632eba90975acb0d4bde43c29 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Thu, 11 Apr 2024 13:23:41 +0200 Subject: [PATCH 04/16] Tweak comment --- client/ayon_core/pipeline/publish/abstract_collect_render.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/publish/abstract_collect_render.py b/client/ayon_core/pipeline/publish/abstract_collect_render.py index 096f1c406e..a9d5b38ada 100644 --- a/client/ayon_core/pipeline/publish/abstract_collect_render.py +++ b/client/ayon_core/pipeline/publish/abstract_collect_render.py @@ -221,7 +221,8 @@ class AbstractCollectRender(pyblish.api.ContextPlugin): instance = context.create_instance(render_instance.name) if render_instance.source_instance is not None: # remove the new instance, because we want to insert it - # at the position of the original instance + # at the position of the original instance to replace the + # source instance in the context completely context.pop() replace_instance_in_context( context, From b3d9dabd62bd98455529ae4fa564c3e6de87379c Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Thu, 11 Apr 2024 13:24:08 +0200 Subject: [PATCH 05/16] Fix returning the instances --- client/ayon_core/hosts/fusion/plugins/publish/collect_render.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/client/ayon_core/hosts/fusion/plugins/publish/collect_render.py b/client/ayon_core/hosts/fusion/plugins/publish/collect_render.py index 521a47675a..7a2844d5db 100644 --- a/client/ayon_core/hosts/fusion/plugins/publish/collect_render.py +++ b/client/ayon_core/hosts/fusion/plugins/publish/collect_render.py @@ -116,6 +116,8 @@ class CollectFusionRender( # to skip ExtractReview locally instance.families.remove("review") + instances.append(instance) + return instances def post_collecting_action(self): From 708fd64c793f41cb423ff7a139072e8f78d73abd Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Thu, 11 Apr 2024 13:29:35 +0200 Subject: [PATCH 06/16] Fix instances - I need more coffee today --- .../hosts/aftereffects/plugins/publish/collect_render.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/client/ayon_core/hosts/aftereffects/plugins/publish/collect_render.py b/client/ayon_core/hosts/aftereffects/plugins/publish/collect_render.py index 46616a2d93..c28042b6ae 100644 --- a/client/ayon_core/hosts/aftereffects/plugins/publish/collect_render.py +++ b/client/ayon_core/hosts/aftereffects/plugins/publish/collect_render.py @@ -146,6 +146,8 @@ class CollectAERender(publish.AbstractCollectRender): # to skip ExtractReview locally instance.families.remove("review") + instances.append(instance) + return instances def get_expected_files(self, render_instance): From 19c655e1532236418b55d07fcb3b44386b8324a5 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Thu, 11 Apr 2024 14:38:28 +0200 Subject: [PATCH 07/16] Update client/ayon_core/pipeline/publish/lib.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- client/ayon_core/pipeline/publish/lib.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index 63fb0feb0a..87d01e5b97 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -937,9 +937,9 @@ def get_publish_instance_families(instance): def replace_instance_in_context( - context: pyblish.api.Context, - source_instance: pyblish.api.Instance, - destination_instance: pyblish.api.Instance + context: pyblish.api.Context, + source_instance: pyblish.api.Instance, + destination_instance: pyblish.api.Instance ): """Replace source instance with the destination instance. From e1bd0ccf540f341905a11862b8b65ce03a94272a Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Thu, 11 Apr 2024 14:38:33 +0200 Subject: [PATCH 08/16] Update client/ayon_core/pipeline/publish/lib.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- client/ayon_core/pipeline/publish/lib.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index 87d01e5b97..2b97feebb3 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -964,9 +964,9 @@ def replace_instance_in_context( destination_instance.data[key] = source_instance.data[key] # Replace the instance at the same index in the context - for i, instance in enumerate(context): + for idx, instance in enumerate(context): if source_instance is instance: - context[i] = destination_instance + context[idx] = destination_instance return True # Source was not found From 25a54cbdca35380195a5998f3b2f34f4561a8c66 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Thu, 11 Apr 2024 14:38:38 +0200 Subject: [PATCH 09/16] Update client/ayon_core/pipeline/publish/lib.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- client/ayon_core/pipeline/publish/lib.py | 1 - 1 file changed, 1 deletion(-) diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index 2b97feebb3..ebdaca5518 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -954,7 +954,6 @@ def replace_instance_in_context( in the context. """ - # Transfer the pyblish.api.Instance id destination_instance._id = source_instance.id From 9690748d993dc2384013fc3a6714cd82e7000435 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Thu, 11 Apr 2024 16:24:40 +0200 Subject: [PATCH 10/16] Do not return `bool` --- client/ayon_core/pipeline/publish/lib.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index ebdaca5518..4cf2c4cb83 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -948,11 +948,6 @@ def replace_instance_in_context( the publisher to correctly detect and transfer the logs relevant for the instance. - Returns: - bool: Whether the source instance was replaced or the destination - instance was only added because source instance did not exist - in the context. - """ # Transfer the pyblish.api.Instance id destination_instance._id = source_instance.id @@ -966,8 +961,7 @@ def replace_instance_in_context( for idx, instance in enumerate(context): if source_instance is instance: context[idx] = destination_instance - return True + return # Source was not found context.append(destination_instance) - return False From 5ad7103f5777f70d244bde459cbf7b98b880f1d9 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Thu, 11 Apr 2024 16:25:34 +0200 Subject: [PATCH 11/16] Raise error if source not found in context --- client/ayon_core/pipeline/publish/lib.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index 4cf2c4cb83..bf733a8620 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -963,5 +963,6 @@ def replace_instance_in_context( context[idx] = destination_instance return - # Source was not found - context.append(destination_instance) + raise ValueError( + f"Source instance {source_instance} not found in context." + ) From 129d5d6b0ed2cbcb1ab673f2e315dc65bfd7471a Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Thu, 11 Apr 2024 17:11:39 +0200 Subject: [PATCH 12/16] Do not create a new instance, just update the existing instance --- .../publish/abstract_collect_render.py | 13 ++----- client/ayon_core/pipeline/publish/lib.py | 34 +------------------ 2 files changed, 4 insertions(+), 43 deletions(-) diff --git a/client/ayon_core/pipeline/publish/abstract_collect_render.py b/client/ayon_core/pipeline/publish/abstract_collect_render.py index a9d5b38ada..d344433d53 100644 --- a/client/ayon_core/pipeline/publish/abstract_collect_render.py +++ b/client/ayon_core/pipeline/publish/abstract_collect_render.py @@ -12,7 +12,6 @@ import six import pyblish.api from .publish_plugins import AbstractMetaContextPlugin -from .lib import replace_instance_in_context @attr.s @@ -218,16 +217,10 @@ class AbstractCollectRender(pyblish.api.ContextPlugin): data = self.add_additional_data(data) render_instance_dict = attr.asdict(render_instance) - instance = context.create_instance(render_instance.name) if render_instance.source_instance is not None: - # remove the new instance, because we want to insert it - # at the position of the original instance to replace the - # source instance in the context completely - context.pop() - replace_instance_in_context( - context, - source_instance=render_instance.source_instance, - destination_instance=instance) + instance = render_instance.source_instance + else: + instance = context.create_instance(render_instance.name) instance.data.update(render_instance_dict) instance.data.update(data) diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index bf733a8620..0be72b459a 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -933,36 +933,4 @@ def get_publish_instance_families(instance): output.append(family) families.discard(family) output.extend(families) - return output - - -def replace_instance_in_context( - context: pyblish.api.Context, - source_instance: pyblish.api.Instance, - destination_instance: pyblish.api.Instance -): - """Replace source instance with the destination instance. - - This transfers the instance's IDs so that the new instance acts exactly - as if it was the source instance the whole time. This is required for - the publisher to correctly detect and transfer the logs relevant for the - instance. - - """ - # Transfer the pyblish.api.Instance id - destination_instance._id = source_instance.id - - # Transfer the `instance_id` of the new publisher's instances - key = "instance_id" - if key in source_instance.data: - destination_instance.data[key] = source_instance.data[key] - - # Replace the instance at the same index in the context - for idx, instance in enumerate(context): - if source_instance is instance: - context[idx] = destination_instance - return - - raise ValueError( - f"Source instance {source_instance} not found in context." - ) + return output \ No newline at end of file From 5f507ff85438654e42891f54daf12a327d87993e Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Thu, 11 Apr 2024 17:12:45 +0200 Subject: [PATCH 13/16] Update comment --- client/ayon_core/pipeline/publish/abstract_collect_render.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/publish/abstract_collect_render.py b/client/ayon_core/pipeline/publish/abstract_collect_render.py index d344433d53..2e56adf66e 100644 --- a/client/ayon_core/pipeline/publish/abstract_collect_render.py +++ b/client/ayon_core/pipeline/publish/abstract_collect_render.py @@ -81,7 +81,7 @@ class RenderInstance(object): outputDir = attr.ib(default=None) context = attr.ib(default=None) - # The source instance this render instance should replace in the context + # The source instance the data of this render instance should merge into source_instance = attr.ib(default=None, type=pyblish.api.Instance) @frameStart.validator From c9dc848b70592088594c8254e256f446101548c5 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Thu, 11 Apr 2024 17:15:42 +0200 Subject: [PATCH 14/16] Cosmetics --- client/ayon_core/pipeline/publish/lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index 0be72b459a..8d3644637b 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -933,4 +933,4 @@ def get_publish_instance_families(instance): output.append(family) families.discard(family) output.extend(families) - return output \ No newline at end of file + return output From c538ed58efd1c35849455822c089dc53b2ca8731 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Thu, 11 Apr 2024 17:26:15 +0200 Subject: [PATCH 15/16] Do not merge `source_instance` into the instance --- .../ayon_core/pipeline/publish/abstract_collect_render.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/ayon_core/pipeline/publish/abstract_collect_render.py b/client/ayon_core/pipeline/publish/abstract_collect_render.py index 2e56adf66e..c50dc16380 100644 --- a/client/ayon_core/pipeline/publish/abstract_collect_render.py +++ b/client/ayon_core/pipeline/publish/abstract_collect_render.py @@ -217,9 +217,9 @@ class AbstractCollectRender(pyblish.api.ContextPlugin): data = self.add_additional_data(data) render_instance_dict = attr.asdict(render_instance) - if render_instance.source_instance is not None: - instance = render_instance.source_instance - else: + # Merge into source instance if provided, otherwise create instance + instance = render_instance_dict.pop("source_instance", None) + if instance is None: instance = context.create_instance(render_instance.name) instance.data.update(render_instance_dict) From cc42c4d5a89978a3ca3be622dc28bbab0e85902f Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 11 Apr 2024 17:52:38 +0200 Subject: [PATCH 16/16] Fix - use differently named variable Original `instance` variable might shadow wanted `render-like` instance if wrong order of instances get processed. --- .../deadline/plugins/publish/submit_fusion_deadline.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/client/ayon_core/modules/deadline/plugins/publish/submit_fusion_deadline.py b/client/ayon_core/modules/deadline/plugins/publish/submit_fusion_deadline.py index bfb65708e6..e3a4cd8030 100644 --- a/client/ayon_core/modules/deadline/plugins/publish/submit_fusion_deadline.py +++ b/client/ayon_core/modules/deadline/plugins/publish/submit_fusion_deadline.py @@ -103,17 +103,17 @@ class FusionSubmitDeadline( # Collect all saver instances in context that are to be rendered saver_instances = [] - for instance in context: - if instance.data["productType"] != "render": + for inst in context: + if inst.data["productType"] != "render": # Allow only saver family instances continue - if not instance.data.get("publish", True): + if not inst.data.get("publish", True): # Skip inactive instances continue - self.log.debug(instance.data["name"]) - saver_instances.append(instance) + self.log.debug(inst.data["name"]) + saver_instances.append(inst) if not saver_instances: raise RuntimeError("No instances found for Deadline submission")