From 922da278e47dcfd3e73c3a58d638d3c615366d57 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 9 Feb 2022 19:24:28 +0100 Subject: [PATCH 1/5] OP-2518 - added create flattened image functionality Groups all publishable layers in additional group, create instance for it with subset name from configurable template. --- .../publish/collect_remote_instances.py | 132 +++++++++++++----- 1 file changed, 95 insertions(+), 37 deletions(-) diff --git a/openpype/hosts/photoshop/plugins/publish/collect_remote_instances.py b/openpype/hosts/photoshop/plugins/publish/collect_remote_instances.py index 48c6f583a4..72279b6358 100644 --- a/openpype/hosts/photoshop/plugins/publish/collect_remote_instances.py +++ b/openpype/hosts/photoshop/plugins/publish/collect_remote_instances.py @@ -14,6 +14,9 @@ class CollectRemoteInstances(pyblish.api.ContextPlugin): Used in remote publishing when artists marks publishable layers by color- coding. + Can add group for all publishable layers to allow creation of flattened + image. (Cannot contain special background layer as it cannot be grouped!) + Identifier: id (str): "pyblish.avalon.instance" """ @@ -26,50 +29,39 @@ class CollectRemoteInstances(pyblish.api.ContextPlugin): # configurable by Settings color_code_mapping = [] + # TODO check if could be set globally, probably doesn't make sense when + # flattened template cannot + subset_template_name = "" + create_flatten_image = False + # probably not possible to configure this globally + flatten_subset_template = "" def process(self, context): self.log.info("CollectRemoteInstances") self.log.debug("mapping:: {}".format(self.color_code_mapping)) - # parse variant if used in webpublishing, comes from webpublisher batch - batch_dir = os.environ.get("OPENPYPE_PUBLISH_DATA") - task_data = None - if batch_dir and os.path.exists(batch_dir): - # TODO check if batch manifest is same as tasks manifests - task_data = parse_json(os.path.join(batch_dir, - "manifest.json")) - if not task_data: - raise ValueError( - "Cannot parse batch meta in {} folder".format(batch_dir)) - variant = task_data["variant"] + existing_subset_names = self._get_existing_subset_names(context) + asset_name, task_name, variant = self._parse_batch() stub = photoshop.stub() layers = stub.get_layers() - existing_subset_names = [] - for instance in context: - if instance.data.get('publish'): - existing_subset_names.append(instance.data.get('subset')) - - asset, task_name, task_type = get_batch_asset_task_info( - task_data["context"]) - - if not task_name: - task_name = task_type - - instance_names = [] + publishable_layers = [] + created_instances = [] + contains_background = False for layer in layers: self.log.debug("Layer:: {}".format(layer)) if layer.parents: self.log.debug("!!! Not a top layer, skip") continue + if not layer.visible: + self.log.debug("Not visible, skip") + continue + resolved_family, resolved_subset_template = self._resolve_mapping( layer ) - self.log.info("resolved_family {}".format(resolved_family)) - self.log.info("resolved_subset_template {}".format( - resolved_subset_template)) if not resolved_subset_template or not resolved_family: self.log.debug("!!! Not found family or template, skip") @@ -90,24 +82,87 @@ class CollectRemoteInstances(pyblish.api.ContextPlugin): "Subset {} already created, skipping.".format(subset)) continue - instance = context.create_instance(layer.name) - instance.append(layer) - instance.data["family"] = resolved_family - instance.data["publish"] = layer.visible - instance.data["asset"] = asset - instance.data["task"] = task_name - instance.data["subset"] = subset + if layer.id == "1": + contains_background = True - instance_names.append(layer.name) + instance = self._create_instance(context, layer, resolved_family, + asset_name, subset, task_name) + existing_subset_names.append(subset) + publishable_layers.append(layer) + created_instances.append(instance) + + if self.create_flatten_image and publishable_layers: + self.log.debug("create_flatten_image") + if not self.flatten_subset_template: + self.log.warning("No template for flatten image") + return + + if contains_background: + raise ValueError("It is not possible to create flatten image " + "with background layer. Please remove it.") + + fill_pairs.pop("layer") + subset = self.flatten_subset_template.format( + **prepare_template_data(fill_pairs)) + + stub.select_layers(publishable_layers) + new_layer = stub.group_selected_layers(subset) + instance = self._create_instance(context, new_layer, + resolved_family, + asset_name, subset, task_name) + created_instances.append(instance) + + for instance in created_instances: # Produce diagnostic message for any graphical # user interface interested in visualising it. self.log.info("Found: \"%s\" " % instance.data["name"]) self.log.info("instance: {} ".format(instance.data)) - if len(instance_names) != len(set(instance_names)): - self.log.warning("Duplicate instances found. " + - "Remove unwanted via SubsetManager") + def _get_existing_subset_names(self, context): + """Collect manually created instances from workfile. + + Shouldn't be any as Webpublisher bypass publishing via Openpype, but + might be some if workfile published through OP is reused. + """ + existing_subset_names = [] + for instance in context: + if instance.data.get('publish'): + existing_subset_names.append(instance.data.get('subset')) + + return existing_subset_names + + def _parse_batch(self): + """Parses asset_name, task_name, variant from batch manifest.""" + batch_dir = os.environ.get("OPENPYPE_PUBLISH_DATA") + task_data = None + if batch_dir and os.path.exists(batch_dir): + task_data = parse_json(os.path.join(batch_dir, + "manifest.json")) + if not task_data: + raise ValueError( + "Cannot parse batch meta in {} folder".format(batch_dir)) + variant = task_data["variant"] + + asset, task_name, task_type = get_batch_asset_task_info( + task_data["context"]) + + if not task_name: + task_name = task_type + + return asset, task_name, variant + + def _create_instance(self, context, layer, family, + asset, subset, task_name): + instance = context.create_instance(layer.name) + instance.append(layer) + instance.data["family"] = family + instance.data["publish"] = True + instance.data["asset"] = asset + instance.data["task"] = task_name + instance.data["subset"] = subset + + return instance def _resolve_mapping(self, layer): """Matches 'layer' color code and name to mapping. @@ -147,4 +202,7 @@ class CollectRemoteInstances(pyblish.api.ContextPlugin): if family_list: family = family_list.pop() + self.log.debug("resolved_family {}".format(family)) + self.log.debug("resolved_subset_template {}".format( + resolved_subset_template)) return family, resolved_subset_template From 697bc11036807d37770e31d894ed8b121368a569 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 9 Feb 2022 23:10:56 +0100 Subject: [PATCH 2/5] OP-2518 - renamed to CollectColorCodedInstances Updated logic for storing layer to instance.data --- openpype/hosts/photoshop/api/ws_stub.py | 19 +++++++++++++- ...es.py => collect_color_coded_instances.py} | 25 +++++++------------ .../plugins/publish/collect_instances.py | 2 +- .../plugins/publish/extract_image.py | 7 +++++- .../plugins/publish/extract_review.py | 2 +- .../defaults/project_settings/photoshop.json | 13 +++------- .../schema_project_photoshop.json | 16 +++++++++--- 7 files changed, 52 insertions(+), 32 deletions(-) rename openpype/hosts/photoshop/plugins/publish/{collect_remote_instances.py => collect_color_coded_instances.py} (91%) diff --git a/openpype/hosts/photoshop/api/ws_stub.py b/openpype/hosts/photoshop/api/ws_stub.py index b8f66332c6..fd8377d4e0 100644 --- a/openpype/hosts/photoshop/api/ws_stub.py +++ b/openpype/hosts/photoshop/api/ws_stub.py @@ -186,9 +186,26 @@ class PhotoshopServerStub: Returns: """ + parent_ids = set([lay.id for lay in layers]) + + return self._get_layers_in_layers(parent_ids) + + def get_layers_in_layers_ids(self, layers_ids): + """Return all layers that belong to layers (might be groups). + + Args: + layers : + + Returns: + + """ + parent_ids = set(layers_ids) + + return self._get_layers_in_layers(parent_ids) + + def _get_layers_in_layers(self, parent_ids): all_layers = self.get_layers() ret = [] - parent_ids = set([lay.id for lay in layers]) for layer in all_layers: parents = set(layer.parents) diff --git a/openpype/hosts/photoshop/plugins/publish/collect_remote_instances.py b/openpype/hosts/photoshop/plugins/publish/collect_color_coded_instances.py similarity index 91% rename from openpype/hosts/photoshop/plugins/publish/collect_remote_instances.py rename to openpype/hosts/photoshop/plugins/publish/collect_color_coded_instances.py index 72279b6358..c1ae88fbbb 100644 --- a/openpype/hosts/photoshop/plugins/publish/collect_remote_instances.py +++ b/openpype/hosts/photoshop/plugins/publish/collect_color_coded_instances.py @@ -8,7 +8,7 @@ from openpype.lib.plugin_tools import parse_json, get_batch_asset_task_info from openpype.hosts.photoshop import api as photoshop -class CollectRemoteInstances(pyblish.api.ContextPlugin): +class CollectColorCodedInstances(pyblish.api.ContextPlugin): """Creates instances for configured color code of a layer. Used in remote publishing when artists marks publishable layers by color- @@ -37,7 +37,7 @@ class CollectRemoteInstances(pyblish.api.ContextPlugin): flatten_subset_template = "" def process(self, context): - self.log.info("CollectRemoteInstances") + self.log.info("CollectColorCodedInstances") self.log.debug("mapping:: {}".format(self.color_code_mapping)) existing_subset_names = self._get_existing_subset_names(context) @@ -48,7 +48,6 @@ class CollectRemoteInstances(pyblish.api.ContextPlugin): publishable_layers = [] created_instances = [] - contains_background = False for layer in layers: self.log.debug("Layer:: {}".format(layer)) if layer.parents: @@ -82,12 +81,8 @@ class CollectRemoteInstances(pyblish.api.ContextPlugin): "Subset {} already created, skipping.".format(subset)) continue - if layer.id == "1": - contains_background = True - instance = self._create_instance(context, layer, resolved_family, asset_name, subset, task_name) - existing_subset_names.append(subset) publishable_layers.append(layer) created_instances.append(instance) @@ -98,19 +93,17 @@ class CollectRemoteInstances(pyblish.api.ContextPlugin): self.log.warning("No template for flatten image") return - if contains_background: - raise ValueError("It is not possible to create flatten image " - "with background layer. Please remove it.") - fill_pairs.pop("layer") subset = self.flatten_subset_template.format( **prepare_template_data(fill_pairs)) - stub.select_layers(publishable_layers) - new_layer = stub.group_selected_layers(subset) - instance = self._create_instance(context, new_layer, - resolved_family, + first_layer = publishable_layers[0] # dummy layer + first_layer.name = subset + family = created_instances[0].data["family"] # inherit family + instance = self._create_instance(context, first_layer, + family, asset_name, subset, task_name) + instance.data["ids"] = [layer.id for layer in publishable_layers] created_instances.append(instance) for instance in created_instances: @@ -155,12 +148,12 @@ class CollectRemoteInstances(pyblish.api.ContextPlugin): def _create_instance(self, context, layer, family, asset, subset, task_name): instance = context.create_instance(layer.name) - instance.append(layer) instance.data["family"] = family instance.data["publish"] = True instance.data["asset"] = asset instance.data["task"] = task_name instance.data["subset"] = subset + instance.data["layer"] = layer return instance diff --git a/openpype/hosts/photoshop/plugins/publish/collect_instances.py b/openpype/hosts/photoshop/plugins/publish/collect_instances.py index f67cc0cbac..c3e27e9646 100644 --- a/openpype/hosts/photoshop/plugins/publish/collect_instances.py +++ b/openpype/hosts/photoshop/plugins/publish/collect_instances.py @@ -43,7 +43,7 @@ class CollectInstances(pyblish.api.ContextPlugin): # continue instance = context.create_instance(layer_data["subset"]) - instance.append(layer) + instance.data["layer"] = layer instance.data.update(layer_data) instance.data["families"] = self.families_mapping[ layer_data["family"] diff --git a/openpype/hosts/photoshop/plugins/publish/extract_image.py b/openpype/hosts/photoshop/plugins/publish/extract_image.py index 2ba81e0bac..beb904215b 100644 --- a/openpype/hosts/photoshop/plugins/publish/extract_image.py +++ b/openpype/hosts/photoshop/plugins/publish/extract_image.py @@ -27,8 +27,13 @@ class ExtractImage(openpype.api.Extractor): self.log.info("Extracting %s" % str(list(instance))) with photoshop.maintained_visibility(): # Hide all other layers. + layer = instance.data.get("layer") + ids = set([layer.id]) + add_ids = instance.data.pop("ids", None) + if add_ids: + ids.update(set(add_ids)) extract_ids = set([ll.id for ll in stub. - get_layers_in_layers([instance[0]])]) + get_layers_in_layers_ids(ids)]) for layer in stub.get_layers(): # limit unnecessary calls to client diff --git a/openpype/hosts/photoshop/plugins/publish/extract_review.py b/openpype/hosts/photoshop/plugins/publish/extract_review.py index 1ad442279a..b6c7e2d189 100644 --- a/openpype/hosts/photoshop/plugins/publish/extract_review.py +++ b/openpype/hosts/photoshop/plugins/publish/extract_review.py @@ -31,7 +31,7 @@ class ExtractReview(openpype.api.Extractor): for image_instance in instance.context: if image_instance.data["family"] != "image": continue - layers.append(image_instance[0]) + layers.append(image_instance.data.get("layer")) # Perform extraction output_image = "{}.jpg".format( diff --git a/openpype/settings/defaults/project_settings/photoshop.json b/openpype/settings/defaults/project_settings/photoshop.json index 31cd815dd8..f095986ee6 100644 --- a/openpype/settings/defaults/project_settings/photoshop.json +++ b/openpype/settings/defaults/project_settings/photoshop.json @@ -7,15 +7,10 @@ } }, "publish": { - "CollectRemoteInstances": { - "color_code_mapping": [ - { - "color_code": [], - "layer_name_regex": [], - "family": "image", - "subset_template_name": "" - } - ] + "CollectColorCodedInstances": { + "create_flatten_image": false, + "flatten_subset_template": "", + "color_code_mapping": [] }, "ValidateContainers": { "enabled": true, diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_photoshop.json b/openpype/settings/entities/schemas/projects_schema/schema_project_photoshop.json index 51ea5b3fe7..f54aa847b5 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_photoshop.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_photoshop.json @@ -37,12 +37,22 @@ "type": "dict", "collapsible": true, "is_group": true, - "key": "CollectRemoteInstances", - "label": "Collect Instances for Webpublish", + "key": "CollectColorCodedInstances", + "label": "Collect Color Coded Instances", "children": [ { "type": "label", - "label": "Set color for publishable layers, set publishable families." + "label": "Set color for publishable layers, set its resulting family and template for subset name. Can create flatten image from published instances" + }, + { + "type": "boolean", + "key": "create_flatten_image", + "label": "Create flatten image" + }, + { + "type": "text", + "key": "flatten_subset_template", + "label": "Subset template for flatten image" }, { "type": "list", From 1a1c65c9126a2f5f2fc1ec68c775ac14ecfd728b Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 11 Feb 2022 11:52:55 +0100 Subject: [PATCH 3/5] OP-2518 - added group and department to submission to DL Group is used to filter on which machines it should be rendered. --- .../deadline/plugins/publish/submit_harmony_deadline.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/openpype/modules/default_modules/deadline/plugins/publish/submit_harmony_deadline.py b/openpype/modules/default_modules/deadline/plugins/publish/submit_harmony_deadline.py index 6683c34dbd..4c6d284c28 100644 --- a/openpype/modules/default_modules/deadline/plugins/publish/submit_harmony_deadline.py +++ b/openpype/modules/default_modules/deadline/plugins/publish/submit_harmony_deadline.py @@ -248,6 +248,8 @@ class HarmonySubmitDeadline( secondary_pool = "" priority = 50 chunk_size = 1000000 + group = "none" + department = "" def get_job_info(self): job_info = DeadlineJobInfo("Harmony") @@ -264,6 +266,8 @@ class HarmonySubmitDeadline( job_info.SecondaryPool = self.secondary_pool job_info.ChunkSize = self.chunk_size job_info.BatchName = os.path.basename(self._instance.data["source"]) + job_info.Department = self.department + job_info.Group = self.group keys = [ "FTRACK_API_KEY", From 85e2a34d3bd9a7330a729c97e85bb6ff7e2a05d3 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 11 Feb 2022 12:04:13 +0100 Subject: [PATCH 4/5] OP-2518 - changed aov_filter to dict-modifiable It should be possible to add new hosts. AOV filter decides if review will be created. --- .../defaults/project_settings/deadline.json | 3 ++ .../schema_project_deadline.json | 41 +++---------------- 2 files changed, 9 insertions(+), 35 deletions(-) diff --git a/openpype/settings/defaults/project_settings/deadline.json b/openpype/settings/defaults/project_settings/deadline.json index f4e46fcddc..6d7f926eae 100644 --- a/openpype/settings/defaults/project_settings/deadline.json +++ b/openpype/settings/defaults/project_settings/deadline.json @@ -36,6 +36,9 @@ ], "celaction": [ ".*" + ], + "harmony": [ + ".*" ] } }, diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_deadline.json b/openpype/settings/entities/schemas/projects_schema/schema_project_deadline.json index 7c44791160..bfd29cf9e8 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_deadline.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_deadline.json @@ -126,43 +126,14 @@ } }, { - "type": "dict", + "type": "dict-modifiable", + "docstring": "Regular expression to filter for which subset review should be created in publish job.", "key": "aov_filter", "label": "Reviewable subsets filter", - "children": [ - { - "type": "list", - "key": "maya", - "label": "Maya", - "object_type": { - "type": "text" - } - }, - { - "type": "list", - "key": "nuke", - "label": "Nuke", - "object_type": { - "type": "text" - } - }, - { - "type": "list", - "key": "aftereffects", - "label": "After Effects", - "object_type": { - "type": "text" - } - }, - { - "type": "list", - "key": "celaction", - "label": "Celaction", - "object_type": { - "type": "text" - } - } - ] + "object_type": { + "type": "list", + "object_type": "text" + } } ] }, From 40003dea40af77b144c1b2e7b2bc2615c57aa334 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 11 Feb 2022 12:05:40 +0100 Subject: [PATCH 5/5] OP-2518 - refactor - switched position, changed label --- .../schema_project_deadline.json | 134 +++++++++--------- 1 file changed, 67 insertions(+), 67 deletions(-) diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_deadline.json b/openpype/settings/entities/schemas/projects_schema/schema_project_deadline.json index bfd29cf9e8..320c80a519 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_deadline.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_deadline.json @@ -71,77 +71,11 @@ } ] }, - { - "type": "dict", - "collapsible": true, - "key": "ProcessSubmittedJobOnFarm", - "label": "ProcessSubmittedJobOnFarm", - "checkbox_key": "enabled", - "is_group": true, - "children": [ - { - "type": "boolean", - "key": "enabled", - "label": "Enabled" - }, - { - "type": "text", - "key": "deadline_department", - "label": "Deadline department" - }, - { - "type": "text", - "key": "deadline_pool", - "label": "Deadline Pool" - }, - { - "type": "text", - "key": "deadline_group", - "label": "Deadline Group" - }, - { - "type": "number", - "key": "deadline_chunk_size", - "label": "Deadline Chunk Size" - }, - { - "type": "number", - "key": "deadline_priority", - "label": "Deadline Priotity" - }, - { - "type": "splitter" - }, - { - "type": "text", - "key": "publishing_script", - "label": "Publishing script path" - }, - { - "type": "list", - "key": "skip_integration_repre_list", - "label": "Skip integration of representation with ext", - "object_type": { - "type": "text" - } - }, - { - "type": "dict-modifiable", - "docstring": "Regular expression to filter for which subset review should be created in publish job.", - "key": "aov_filter", - "label": "Reviewable subsets filter", - "object_type": { - "type": "list", - "object_type": "text" - } - } - ] - }, { "type": "dict", "collapsible": true, "key": "MayaSubmitDeadline", - "label": "Submit Maya job to Deadline", + "label": "Maya Submit to Deadline", "checkbox_key": "enabled", "children": [ { @@ -435,6 +369,72 @@ "label": "Department" } ] + }, + { + "type": "dict", + "collapsible": true, + "key": "ProcessSubmittedJobOnFarm", + "label": "ProcessSubmittedJobOnFarm", + "checkbox_key": "enabled", + "is_group": true, + "children": [ + { + "type": "boolean", + "key": "enabled", + "label": "Enabled" + }, + { + "type": "text", + "key": "deadline_department", + "label": "Deadline department" + }, + { + "type": "text", + "key": "deadline_pool", + "label": "Deadline Pool" + }, + { + "type": "text", + "key": "deadline_group", + "label": "Deadline Group" + }, + { + "type": "number", + "key": "deadline_chunk_size", + "label": "Deadline Chunk Size" + }, + { + "type": "number", + "key": "deadline_priority", + "label": "Deadline Priotity" + }, + { + "type": "splitter" + }, + { + "type": "text", + "key": "publishing_script", + "label": "Publishing script path" + }, + { + "type": "list", + "key": "skip_integration_repre_list", + "label": "Skip integration of representation with ext", + "object_type": { + "type": "text" + } + }, + { + "type": "dict-modifiable", + "docstring": "Regular expression to filter for which subset review should be created in publish job.", + "key": "aov_filter", + "label": "Reviewable subsets filter", + "object_type": { + "type": "list", + "object_type": "text" + } + } + ] } ] }