From 551f34a873c89e739dc0b5d28a74eeec3f79dac2 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Fri, 12 Aug 2022 14:25:29 +0200 Subject: [PATCH 001/102] Add subsetGroup column to scene inventory --- openpype/tools/sceneinventory/model.py | 11 +++++++++-- openpype/tools/sceneinventory/window.py | 3 ++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/openpype/tools/sceneinventory/model.py b/openpype/tools/sceneinventory/model.py index 63fbe04c5c..97cc11ff23 100644 --- a/openpype/tools/sceneinventory/model.py +++ b/openpype/tools/sceneinventory/model.py @@ -34,7 +34,8 @@ from .lib import ( class InventoryModel(TreeModel): """The model for the inventory""" - Columns = ["Name", "version", "count", "family", "loader", "objectName"] + Columns = ["Name", "version", "count", "family", + "subsetGroup", "loader", "objectName"] OUTDATED_COLOR = QtGui.QColor(235, 30, 30) CHILD_OUTDATED_COLOR = QtGui.QColor(200, 160, 30) @@ -157,8 +158,13 @@ class InventoryModel(TreeModel): # Family icon return item.get("familyIcon", None) + column_name = self.Columns[index.column()] + + if column_name == "subsetGroup" and item.get("subsetGroup"): + return qtawesome.icon("fa.object-group", + color=get_default_entity_icon_color()) + if item.get("isGroupNode"): - column_name = self.Columns[index.column()] if column_name == "active_site": provider = item.get("active_site_provider") return self._site_icons.get(provider) @@ -423,6 +429,7 @@ class InventoryModel(TreeModel): group_node["familyIcon"] = family_icon group_node["count"] = len(group_items) group_node["isGroupNode"] = True + group_node["subsetGroup"] = subset["data"].get("subsetGroup") if self.sync_enabled: progress = get_progress_for_repre( diff --git a/openpype/tools/sceneinventory/window.py b/openpype/tools/sceneinventory/window.py index 054c2a2daa..02addbccfe 100644 --- a/openpype/tools/sceneinventory/window.py +++ b/openpype/tools/sceneinventory/window.py @@ -88,7 +88,8 @@ class SceneInventoryWindow(QtWidgets.QDialog): view.setColumnWidth(1, 55) # version view.setColumnWidth(2, 55) # count view.setColumnWidth(3, 150) # family - view.setColumnWidth(4, 100) # namespace + view.setColumnWidth(4, 120) # subsetGroup + view.setColumnWidth(5, 150) # loader # apply delegates version_delegate = VersionDelegate(legacy_io, self) From 2cebaf718cc3c7df21a3543e90bac91ae56e68d8 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 15 Aug 2022 21:04:51 +0200 Subject: [PATCH 002/102] global: moving collect audio to global --- .../plugins/publish/collect_audio.py | 113 ------------------ openpype/plugins/publish/collect_audio.py | 51 ++++++++ 2 files changed, 51 insertions(+), 113 deletions(-) delete mode 100644 openpype/hosts/celaction/plugins/publish/collect_audio.py create mode 100644 openpype/plugins/publish/collect_audio.py diff --git a/openpype/hosts/celaction/plugins/publish/collect_audio.py b/openpype/hosts/celaction/plugins/publish/collect_audio.py deleted file mode 100644 index c6e3bf2c03..0000000000 --- a/openpype/hosts/celaction/plugins/publish/collect_audio.py +++ /dev/null @@ -1,113 +0,0 @@ -import os -import collections -from pprint import pformat - -import pyblish.api - -from openpype.client import ( - get_subsets, - get_last_versions, - get_representations -) -from openpype.pipeline import legacy_io - - -class AppendCelactionAudio(pyblish.api.ContextPlugin): - - label = "Colect Audio for publishing" - order = pyblish.api.CollectorOrder + 0.1 - - def process(self, context): - self.log.info('Collecting Audio Data') - asset_doc = context.data["assetEntity"] - - # get all available representations - subsets = self.get_subsets( - asset_doc, - representations=["audio", "wav"] - ) - self.log.info(f"subsets is: {pformat(subsets)}") - - if not subsets.get("audioMain"): - raise AttributeError("`audioMain` subset does not exist") - - reprs = subsets.get("audioMain", {}).get("representations", []) - self.log.info(f"reprs is: {pformat(reprs)}") - - repr = next((r for r in reprs), None) - if not repr: - raise "Missing `audioMain` representation" - self.log.info(f"representation is: {repr}") - - audio_file = repr.get('data', {}).get('path', "") - - if os.path.exists(audio_file): - context.data["audioFile"] = audio_file - self.log.info( - 'audio_file: {}, has been added to context'.format(audio_file)) - else: - self.log.warning("Couldn't find any audio file on Ftrack.") - - def get_subsets(self, asset_doc, representations): - """ - Query subsets with filter on name. - - The method will return all found subsets and its defined version - and subsets. Version could be specified with number. Representation - can be filtered. - - Arguments: - asset_doct (dict): Asset (shot) mongo document - representations (list): list for all representations - - Returns: - dict: subsets with version and representations in keys - """ - - # Query all subsets for asset - project_name = legacy_io.active_project() - subset_docs = get_subsets( - project_name, asset_ids=[asset_doc["_id"]], fields=["_id"] - ) - # Collect all subset ids - subset_ids = [ - subset_doc["_id"] - for subset_doc in subset_docs - ] - - # Check if we found anything - assert subset_ids, ( - "No subsets found. Check correct filter. " - "Try this for start `r'.*'`: asset: `{}`" - ).format(asset_doc["name"]) - - last_versions_by_subset_id = get_last_versions( - project_name, subset_ids, fields=["_id", "parent"] - ) - - version_docs_by_id = {} - for version_doc in last_versions_by_subset_id.values(): - version_docs_by_id[version_doc["_id"]] = version_doc - - repre_docs = get_representations( - project_name, - version_ids=version_docs_by_id.keys(), - representation_names=representations - ) - repre_docs_by_version_id = collections.defaultdict(list) - for repre_doc in repre_docs: - version_id = repre_doc["parent"] - repre_docs_by_version_id[version_id].append(repre_doc) - - output_dict = {} - for version_id, repre_docs in repre_docs_by_version_id.items(): - version_doc = version_docs_by_id[version_id] - subset_id = version_doc["parent"] - subset_doc = last_versions_by_subset_id[subset_id] - # Store queried docs by subset name - output_dict[subset_doc["name"]] = { - "representations": repre_docs, - "version": version_doc - } - - return output_dict diff --git a/openpype/plugins/publish/collect_audio.py b/openpype/plugins/publish/collect_audio.py new file mode 100644 index 0000000000..022334e0f3 --- /dev/null +++ b/openpype/plugins/publish/collect_audio.py @@ -0,0 +1,51 @@ +import pyblish.api +from pprint import pformat + +from openpype.client import ( + get_last_version_by_subset_name, + get_representations, +) +from openpype.pipeline import ( + legacy_io, + get_representation_path, +) + + +class CollectAudio(pyblish.api.InstancePlugin): + + label = "Colect Audio" + order = pyblish.api.CollectorOrder + 0.1 + hosts = ["standalonepublisher"] + + def process(self, instance): + self.log.info('Collecting Audio Data') + + project_name = legacy_io.active_project() + asset_name = instance.data["asset"] + # * Add audio to instance if exists. + # Find latest versions document + last_version_doc = get_last_version_by_subset_name( + project_name, "audioMain", asset_name=asset_name, fields=["_id"] + ) + + repre_doc = None + if last_version_doc: + # Try to find it's representation (Expected there is only one) + repre_docs = list(get_representations( + project_name, version_ids=[last_version_doc["_id"]] + )) + if not repre_docs: + self.log.warning( + "Version document does not contain any representations" + ) + else: + repre_doc = repre_docs[0] + + # Add audio to instance if representation was found + if repre_doc: + instance.data["audio"] = [{ + "offset": 0, + "filename": get_representation_path(repre_doc) + }] + + self.log.debug("instance.data: {}".format(pformat(instance.data))) From 7c06a1fe8cbbdd612fb9a6a8f6b8df092f8e810a Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 15 Aug 2022 21:06:40 +0200 Subject: [PATCH 003/102] global: improving hosts and families in collect audio --- openpype/plugins/publish/collect_audio.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/openpype/plugins/publish/collect_audio.py b/openpype/plugins/publish/collect_audio.py index 022334e0f3..f4cad86f94 100644 --- a/openpype/plugins/publish/collect_audio.py +++ b/openpype/plugins/publish/collect_audio.py @@ -15,7 +15,24 @@ class CollectAudio(pyblish.api.InstancePlugin): label = "Colect Audio" order = pyblish.api.CollectorOrder + 0.1 - hosts = ["standalonepublisher"] + families = ["review"] + hosts = [ + "nuke", + "maya", + "shell", + "hiero", + "premiere", + "harmony", + "traypublisher", + "standalonepublisher", + "fusion", + "tvpaint", + "resolve", + "webpublisher", + "aftereffects", + "flame", + "unreal" + ] def process(self, instance): self.log.info('Collecting Audio Data') From 495a65707cfa7384dcd1498863a34942cb85a5ac Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 15 Aug 2022 21:13:09 +0200 Subject: [PATCH 004/102] Global: improving docstring and comments --- openpype/plugins/publish/collect_audio.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/openpype/plugins/publish/collect_audio.py b/openpype/plugins/publish/collect_audio.py index f4cad86f94..4ba47f739d 100644 --- a/openpype/plugins/publish/collect_audio.py +++ b/openpype/plugins/publish/collect_audio.py @@ -12,7 +12,9 @@ from openpype.pipeline import ( class CollectAudio(pyblish.api.InstancePlugin): + """ Collecting available audio subset to instance + """ label = "Colect Audio" order = pyblish.api.CollectorOrder + 0.1 families = ["review"] @@ -35,11 +37,12 @@ class CollectAudio(pyblish.api.InstancePlugin): ] def process(self, instance): + # * Add audio to instance if exists. self.log.info('Collecting Audio Data') project_name = legacy_io.active_project() asset_name = instance.data["asset"] - # * Add audio to instance if exists. + # Find latest versions document last_version_doc = get_last_version_by_subset_name( project_name, "audioMain", asset_name=asset_name, fields=["_id"] From 74934a51b9ce7d581473426ef9206fad1ab4b486 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 16 Aug 2022 17:21:37 +0200 Subject: [PATCH 005/102] Nuke: removing audio inclusion from precollect write --- .../nuke/plugins/publish/precollect_writes.py | 28 ------------------- 1 file changed, 28 deletions(-) diff --git a/openpype/hosts/nuke/plugins/publish/precollect_writes.py b/openpype/hosts/nuke/plugins/publish/precollect_writes.py index e37cc8a80a..17c4bc30cf 100644 --- a/openpype/hosts/nuke/plugins/publish/precollect_writes.py +++ b/openpype/hosts/nuke/plugins/publish/precollect_writes.py @@ -201,34 +201,6 @@ class CollectNukeWrites(pyblish.api.InstancePlugin): if not instance.data["review"]: instance.data["useSequenceForReview"] = False - project_name = legacy_io.active_project() - asset_name = instance.data["asset"] - # * Add audio to instance if exists. - # Find latest versions document - last_version_doc = get_last_version_by_subset_name( - project_name, "audioMain", asset_name=asset_name, fields=["_id"] - ) - - repre_doc = None - if last_version_doc: - # Try to find it's representation (Expected there is only one) - repre_docs = list(get_representations( - project_name, version_ids=[last_version_doc["_id"]] - )) - if not repre_docs: - self.log.warning( - "Version document does not contain any representations" - ) - else: - repre_doc = repre_docs[0] - - # Add audio to instance if representation was found - if repre_doc: - instance.data["audio"] = [{ - "offset": 0, - "filename": get_representation_path(repre_doc) - }] - self.log.debug("instance.data: {}".format(pformat(instance.data))) def is_prerender(self, families): From 737edadfd5166eb20197d818eefe2a1a22041b0d Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 16 Aug 2022 17:23:42 +0200 Subject: [PATCH 006/102] global: preparation for settings attribute --- openpype/plugins/publish/collect_audio.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/openpype/plugins/publish/collect_audio.py b/openpype/plugins/publish/collect_audio.py index 4ba47f739d..7e3b42f375 100644 --- a/openpype/plugins/publish/collect_audio.py +++ b/openpype/plugins/publish/collect_audio.py @@ -36,6 +36,8 @@ class CollectAudio(pyblish.api.InstancePlugin): "unreal" ] + audio_subset_name = "audioMain" + def process(self, instance): # * Add audio to instance if exists. self.log.info('Collecting Audio Data') @@ -45,7 +47,10 @@ class CollectAudio(pyblish.api.InstancePlugin): # Find latest versions document last_version_doc = get_last_version_by_subset_name( - project_name, "audioMain", asset_name=asset_name, fields=["_id"] + project_name, + self.audio_subset_name, + asset_name=asset_name, + fields=["_id"] ) repre_doc = None From 266975d6942156e71565a3847e4e62a9490dee71 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 16 Aug 2022 17:36:39 +0200 Subject: [PATCH 007/102] settings: adding collect audio plugin --- .../defaults/project_settings/global.json | 4 ++++ .../schemas/schema_global_publish.json | 21 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/openpype/settings/defaults/project_settings/global.json b/openpype/settings/defaults/project_settings/global.json index 0ff9363ba7..9258343440 100644 --- a/openpype/settings/defaults/project_settings/global.json +++ b/openpype/settings/defaults/project_settings/global.json @@ -3,6 +3,10 @@ "CollectAnatomyInstanceData": { "follow_workfile_version": false }, + "CollectAudio": { + "enabled": true, + "audio_subset_name": "audioMain" + }, "CollectSceneVersion": { "hosts": [ "aftereffects", diff --git a/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_publish.json b/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_publish.json index e1aa230b49..2efee92832 100644 --- a/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_publish.json +++ b/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_publish.json @@ -18,6 +18,27 @@ } ] }, + { + "type": "dict", + "collapsible": true, + "checkbox_key": "enabled", + "key": "CollectAudio", + "label": "Collect Audio", + "is_group": true, + "children": [ + { + "type": "boolean", + "key": "enabled", + "label": "Enabled" + }, + { + "key": "audio_subset_name", + "label": "Name of audio variant", + "type": "text", + "placeholder": "audioMain" + } + ] + }, { "type": "dict", "collapsible": true, From 98ac7f538e6482007115aff917cdf4ccd39fbc83 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 17 Aug 2022 11:59:15 +0200 Subject: [PATCH 008/102] condition for case where audio is already collected --- openpype/plugins/publish/collect_audio.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/openpype/plugins/publish/collect_audio.py b/openpype/plugins/publish/collect_audio.py index 7e3b42f375..6aed3f82fe 100644 --- a/openpype/plugins/publish/collect_audio.py +++ b/openpype/plugins/publish/collect_audio.py @@ -68,9 +68,10 @@ class CollectAudio(pyblish.api.InstancePlugin): # Add audio to instance if representation was found if repre_doc: - instance.data["audio"] = [{ - "offset": 0, - "filename": get_representation_path(repre_doc) - }] + if not instance.data.get("audio"): + instance.data["audio"] = [{ + "offset": 0, + "filename": get_representation_path(repre_doc) + }] self.log.debug("instance.data: {}".format(pformat(instance.data))) From ed2aedd0feec05d7d53f2e62789f77838a6f2f47 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 18 Aug 2022 10:17:02 +0200 Subject: [PATCH 009/102] processing PR comments --- openpype/plugins/publish/collect_audio.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/openpype/plugins/publish/collect_audio.py b/openpype/plugins/publish/collect_audio.py index 6aed3f82fe..cf074392ee 100644 --- a/openpype/plugins/publish/collect_audio.py +++ b/openpype/plugins/publish/collect_audio.py @@ -15,7 +15,7 @@ class CollectAudio(pyblish.api.InstancePlugin): """ Collecting available audio subset to instance """ - label = "Colect Audio" + label = "Collect Audio" order = pyblish.api.CollectorOrder + 0.1 families = ["review"] hosts = [ @@ -39,8 +39,14 @@ class CollectAudio(pyblish.api.InstancePlugin): audio_subset_name = "audioMain" def process(self, instance): - # * Add audio to instance if exists. - self.log.info('Collecting Audio Data') + if instance.data.get("audio"): + self.log.info( + "Skipping Audio collecion. It is already collected" + ) + return + + # Add audio to instance if exists. + self.log.info('Collecting Audio Data ...') project_name = legacy_io.active_project() asset_name = instance.data["asset"] @@ -68,10 +74,10 @@ class CollectAudio(pyblish.api.InstancePlugin): # Add audio to instance if representation was found if repre_doc: - if not instance.data.get("audio"): - instance.data["audio"] = [{ - "offset": 0, - "filename": get_representation_path(repre_doc) - }] + instance.data["audio"] = [{ + "offset": 0, + "filename": get_representation_path(repre_doc) + }] + self.log.info("Audio Data added to instance ...") self.log.debug("instance.data: {}".format(pformat(instance.data))) From 694a07579287b12b5fcce35fc4a405700ca3f64c Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Thu, 18 Aug 2022 11:45:01 +0200 Subject: [PATCH 010/102] Refactor `subsetGroup` column name to `group` Co-authored-by: Milan Kolar --- openpype/tools/sceneinventory/model.py | 6 +++--- openpype/tools/sceneinventory/window.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/openpype/tools/sceneinventory/model.py b/openpype/tools/sceneinventory/model.py index 97cc11ff23..1a3b7c7055 100644 --- a/openpype/tools/sceneinventory/model.py +++ b/openpype/tools/sceneinventory/model.py @@ -35,7 +35,7 @@ class InventoryModel(TreeModel): """The model for the inventory""" Columns = ["Name", "version", "count", "family", - "subsetGroup", "loader", "objectName"] + "group", "loader", "objectName"] OUTDATED_COLOR = QtGui.QColor(235, 30, 30) CHILD_OUTDATED_COLOR = QtGui.QColor(200, 160, 30) @@ -160,7 +160,7 @@ class InventoryModel(TreeModel): column_name = self.Columns[index.column()] - if column_name == "subsetGroup" and item.get("subsetGroup"): + if column_name == "group" and item.get("group"): return qtawesome.icon("fa.object-group", color=get_default_entity_icon_color()) @@ -429,7 +429,7 @@ class InventoryModel(TreeModel): group_node["familyIcon"] = family_icon group_node["count"] = len(group_items) group_node["isGroupNode"] = True - group_node["subsetGroup"] = subset["data"].get("subsetGroup") + group_node["group"] = subset["data"].get("subsetGroup") if self.sync_enabled: progress = get_progress_for_repre( diff --git a/openpype/tools/sceneinventory/window.py b/openpype/tools/sceneinventory/window.py index 02addbccfe..1f4585b650 100644 --- a/openpype/tools/sceneinventory/window.py +++ b/openpype/tools/sceneinventory/window.py @@ -88,7 +88,7 @@ class SceneInventoryWindow(QtWidgets.QDialog): view.setColumnWidth(1, 55) # version view.setColumnWidth(2, 55) # count view.setColumnWidth(3, 150) # family - view.setColumnWidth(4, 120) # subsetGroup + view.setColumnWidth(4, 120) # group view.setColumnWidth(5, 150) # loader # apply delegates From 149b65b8b05750966ab5d93a5751e7085cf81652 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 23 Aug 2022 16:14:38 +0200 Subject: [PATCH 011/102] global: audio PR comments --- openpype/plugins/publish/collect_audio.py | 45 +++++++++++++++++------ 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/openpype/plugins/publish/collect_audio.py b/openpype/plugins/publish/collect_audio.py index cf074392ee..3a765d345d 100644 --- a/openpype/plugins/publish/collect_audio.py +++ b/openpype/plugins/publish/collect_audio.py @@ -12,10 +12,12 @@ from openpype.pipeline import ( class CollectAudio(pyblish.api.InstancePlugin): - """ Collecting available audio subset to instance + """Collect asset's last published audio. + The audio subset name searched for is defined in: + project settings > Collect Audio """ - label = "Collect Audio" + label = "Collect Asset Audio" order = pyblish.api.CollectorOrder + 0.1 families = ["review"] hosts = [ @@ -46,10 +48,33 @@ class CollectAudio(pyblish.api.InstancePlugin): return # Add audio to instance if exists. - self.log.info('Collecting Audio Data ...') + self.log.info(( + "Searching for audio subset '{subset}'" + " in asset '{asset}'" + ).format( + subset=self.audio_subset_name, + asset=instance.data["asset"] + )) + + repre_doc = self._get_repre_doc(instance) + + # Add audio to instance if representation was found + if repre_doc: + instance.data["audio"] = [{ + "offset": 0, + "filename": get_representation_path(repre_doc) + }] + self.log.info("Audio Data added to instance ...") + + def _get_repre_doc(self, instance): + cache = instance.context.data.get("__cache_asset_audio", {}) + asset_name = instance.data["asset"] + + # first try to get it from cache + if asset_name in cache: + return cache[asset_name] project_name = legacy_io.active_project() - asset_name = instance.data["asset"] # Find latest versions document last_version_doc = get_last_version_by_subset_name( @@ -72,12 +97,8 @@ class CollectAudio(pyblish.api.InstancePlugin): else: repre_doc = repre_docs[0] - # Add audio to instance if representation was found - if repre_doc: - instance.data["audio"] = [{ - "offset": 0, - "filename": get_representation_path(repre_doc) - }] - self.log.info("Audio Data added to instance ...") + # update cache + cache[asset_name] = repre_doc + instance.context.data["__cache_asset_audio"].update(cache) - self.log.debug("instance.data: {}".format(pformat(instance.data))) + return repre_doc From d2d90ed2e098587cd466243f5d666093ed1db55f Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 23 Aug 2022 17:30:43 +0200 Subject: [PATCH 012/102] hound catch --- openpype/plugins/publish/collect_audio.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openpype/plugins/publish/collect_audio.py b/openpype/plugins/publish/collect_audio.py index 3a765d345d..e2fb766ec4 100644 --- a/openpype/plugins/publish/collect_audio.py +++ b/openpype/plugins/publish/collect_audio.py @@ -1,5 +1,4 @@ import pyblish.api -from pprint import pformat from openpype.client import ( get_last_version_by_subset_name, From 65d785d100c986128a88f1dc2c77b5321b85d7da Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Wed, 24 Aug 2022 00:29:08 +0200 Subject: [PATCH 013/102] Draft stash for refactoring maya submit deadline to use `AbstractSubmitDeadline` base. - This does *NOT* work currently! --- .../deadline/abstract_submit_deadline.py | 22 + .../plugins/publish/submit_maya_deadline.py | 1297 ++++++++--------- 2 files changed, 623 insertions(+), 696 deletions(-) diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py index 0bad981fdf..577378335e 100644 --- a/openpype/modules/deadline/abstract_submit_deadline.py +++ b/openpype/modules/deadline/abstract_submit_deadline.py @@ -359,6 +359,27 @@ class DeadlineJobInfo(object): def OutputDirectory(self, val): # noqa: N802 self._outputDirectory.append(val) + # Asset Dependency + # ---------------------------------------------- + _assetDependency = attr.ib(factory=list) + + @property + def AssetDependency(self): # noqa: N802 + """Return all OutputDirectory values formatted for Deadline. + + Returns: + dict: as `{'OutputDirectory0': 'dir'}` + + """ + out = {} + for index, v in enumerate(self._assetDependency): + out["AssetDependency{}".format(index)] = v + return out + + @OutputDirectory.setter + def AssetDependency(self, val): # noqa: N802 + self._assetDependency.append(val) + # Tile Job # ---------------------------------------------- TileJob = attr.ib(default=None) # Default: false @@ -396,6 +417,7 @@ class DeadlineJobInfo(object): serialized.update(self.OutputFilename) serialized.update(self.OutputFilenameTile) serialized.update(self.OutputDirectory) + serialized.update(self.AssetDependency) return serialized diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 7966861358..6dfa48a9f8 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -18,7 +18,6 @@ Attributes: from __future__ import print_function import os -import json import getpass import copy import re @@ -27,252 +26,32 @@ from datetime import datetime import itertools from collections import OrderedDict +import attr import clique -import requests from maya import cmds -import pyblish.api - -from openpype.lib import requests_post from openpype.hosts.maya.api import lib from openpype.pipeline import legacy_io -# Documentation for keys available at: -# https://docs.thinkboxsoftware.com -# /products/deadline/8.0/1_User%20Manual/manual -# /manual-submission.html#job-info-file-options - -payload_skeleton_template = { - "JobInfo": { - "BatchName": None, # Top-level group name - "Name": None, # Job name, as seen in Monitor - "UserName": None, - "Plugin": "MayaBatch", - "Frames": "{start}-{end}x{step}", - "Comment": None, - "Priority": 50, - }, - "PluginInfo": { - "SceneFile": None, # Input - "OutputFilePath": None, # Output directory and filename - "OutputFilePrefix": None, - "Version": cmds.about(version=True), # Mandatory for Deadline - "UsingRenderLayers": True, - "RenderLayer": None, # Render only this layer - "Renderer": None, - "ProjectPath": None, # Resolve relative references - "RenderSetupIncludeLights": None, # Include all lights flag. - }, - "AuxFiles": [] # Mandatory for Deadline, may be empty -} +from openpype_modules.deadline import abstract_submit_deadline +from openpype_modules.deadline.abstract_submit_deadline import DeadlineJobInfo -def _format_tiles( - filename, index, tiles_x, tiles_y, - width, height, prefix): - """Generate tile entries for Deadline tile job. - - Returns two dictionaries - one that can be directly used in Deadline - job, second that can be used for Deadline Assembly job configuration - file. - - This will format tile names: - - Example:: - { - "OutputFilename0Tile0": "_tile_1x1_4x4_Main_beauty.1001.exr", - "OutputFilename0Tile1": "_tile_2x1_4x4_Main_beauty.1001.exr" - } - - And add tile prefixes like: - - Example:: - Image prefix is: - `maya///_` - - Result for tile 0 for 4x4 will be: - `maya///_tile_1x1_4x4__` - - Calculating coordinates is tricky as in Job they are defined as top, - left, bottom, right with zero being in top-left corner. But Assembler - configuration file takes tile coordinates as X, Y, Width and Height and - zero is bottom left corner. - - Args: - filename (str): Filename to process as tiles. - index (int): Index of that file if it is sequence. - tiles_x (int): Number of tiles in X. - tiles_y (int): Number if tikes in Y. - width (int): Width resolution of final image. - height (int): Height resolution of final image. - prefix (str): Image prefix. - - Returns: - (dict, dict): Tuple of two dictionaires - first can be used to - extend JobInfo, second has tiles x, y, width and height - used for assembler configuration. - - """ - tile = 0 - out = {"JobInfo": {}, "PluginInfo": {}} - cfg = OrderedDict() - w_space = width / tiles_x - h_space = height / tiles_y - - cfg["TilesCropped"] = "False" - - for tile_x in range(1, tiles_x + 1): - for tile_y in reversed(range(1, tiles_y + 1)): - tile_prefix = "_tile_{}x{}_{}x{}_".format( - tile_x, tile_y, - tiles_x, - tiles_y - ) - out_tile_index = "OutputFilename{}Tile{}".format( - str(index), tile - ) - new_filename = "{}/{}{}".format( - os.path.dirname(filename), - tile_prefix, - os.path.basename(filename) - ) - out["JobInfo"][out_tile_index] = new_filename - out["PluginInfo"]["RegionPrefix{}".format(str(tile))] = \ - "/{}".format(tile_prefix).join(prefix.rsplit("/", 1)) - - out["PluginInfo"]["RegionTop{}".format(tile)] = int(height) - (tile_y * h_space) # noqa: E501 - out["PluginInfo"]["RegionBottom{}".format(tile)] = int(height) - ((tile_y - 1) * h_space) - 1 # noqa: E501 - out["PluginInfo"]["RegionLeft{}".format(tile)] = (tile_x - 1) * w_space # noqa: E501 - out["PluginInfo"]["RegionRight{}".format(tile)] = (tile_x * w_space) - 1 # noqa: E501 - - cfg["Tile{}".format(tile)] = new_filename - cfg["Tile{}Tile".format(tile)] = new_filename - cfg["Tile{}FileName".format(tile)] = new_filename - cfg["Tile{}X".format(tile)] = (tile_x - 1) * w_space - - cfg["Tile{}Y".format(tile)] = int(height) - (tile_y * h_space) - - cfg["Tile{}Width".format(tile)] = w_space - cfg["Tile{}Height".format(tile)] = h_space - - tile += 1 - return out, cfg +@attr.s +class DeadlinePluginInfo(): + SceneFile = attr.ib(default=None) # Input + OutputFilePath = attr.ib(default=None) # Output directory and filename + OutputFilePrefix = attr.ib(default=None) + Version = attr.ib(default=None) # Mandatory for Deadline + UsingRenderLayers = attr.ib(default=True) + RenderLayer = attr.ib(default=None) # Render only this layer + Renderer = attr.ib(default=None) + ProjectPath = attr.ib(default=None) # Resolve relative references + RenderSetupIncludeLights = attr.ib(default=None) # Include all lights flag -def get_renderer_variables(renderlayer, root): - """Retrieve the extension which has been set in the VRay settings. - - Will return None if the current renderer is not VRay - For Maya 2016.5 and up the renderSetup creates renderSetupLayer node which - start with `rs`. Use the actual node name, do NOT use the `nice name` - - Args: - renderlayer (str): the node name of the renderlayer. - root (str): base path to render - - Returns: - dict - - """ - renderer = lib.get_renderer(renderlayer or lib.get_current_renderlayer()) - render_attrs = lib.RENDER_ATTRS.get(renderer, lib.RENDER_ATTRS["default"]) - - padding = cmds.getAttr("{}.{}".format(render_attrs["node"], - render_attrs["padding"])) - - filename_0 = cmds.renderSettings( - fullPath=True, - gin="#" * int(padding), - lut=True, - layer=renderlayer or lib.get_current_renderlayer())[0] - filename_0 = re.sub('_', '_beauty', - filename_0, flags=re.IGNORECASE) - prefix_attr = "defaultRenderGlobals.imageFilePrefix" - - scene = cmds.file(query=True, sceneName=True) - scene, _ = os.path.splitext(os.path.basename(scene)) - - if renderer == "vray": - renderlayer = renderlayer.split("_")[-1] - # Maya's renderSettings function does not return V-Ray file extension - # so we get the extension from vraySettings - extension = cmds.getAttr("vraySettings.imageFormatStr") - - # When V-Ray image format has not been switched once from default .png - # the getAttr command above returns None. As such we explicitly set - # it to `.png` - if extension is None: - extension = "png" - - if extension in ["exr (multichannel)", "exr (deep)"]: - extension = "exr" - - prefix_attr = "vraySettings.fileNamePrefix" - filename_prefix = cmds.getAttr(prefix_attr) - # we need to determine path for vray as maya `renderSettings` query - # does not work for vray. - - filename_0 = re.sub('', scene, filename_prefix, flags=re.IGNORECASE) # noqa: E501 - filename_0 = re.sub('', renderlayer, filename_0, flags=re.IGNORECASE) # noqa: E501 - filename_0 = "{}.{}.{}".format( - filename_0, "#" * int(padding), extension) - filename_0 = os.path.normpath(os.path.join(root, filename_0)) - elif renderer == "renderman": - prefix_attr = "rmanGlobals.imageFileFormat" - # NOTE: This is guessing extensions from renderman display types. - # Some of them are just framebuffers, d_texture format can be - # set in display setting. We set those now to None, but it - # should be handled more gracefully. - display_types = { - "d_deepexr": "exr", - "d_it": None, - "d_null": None, - "d_openexr": "exr", - "d_png": "png", - "d_pointcloud": "ptc", - "d_targa": "tga", - "d_texture": None, - "d_tiff": "tif" - } - - extension = display_types.get( - cmds.listConnections("rmanDefaultDisplay.displayType")[0], - "exr" - ) or "exr" - - filename_prefix = "{}/{}".format( - cmds.getAttr("rmanGlobals.imageOutputDir"), - cmds.getAttr("rmanGlobals.imageFileFormat") - ) - - renderlayer = renderlayer.split("_")[-1] - - filename_0 = re.sub('', scene, filename_prefix, flags=re.IGNORECASE) # noqa: E501 - filename_0 = re.sub('', renderlayer, filename_0, flags=re.IGNORECASE) # noqa: E501 - filename_0 = re.sub('', "#" * int(padding), filename_0, flags=re.IGNORECASE) # noqa: E501 - filename_0 = re.sub('', extension, filename_0, flags=re.IGNORECASE) # noqa: E501 - filename_0 = os.path.normpath(os.path.join(root, filename_0)) - elif renderer == "redshift": - # mapping redshift extension dropdown values to strings - ext_mapping = ["iff", "exr", "tif", "png", "tga", "jpg"] - extension = ext_mapping[ - cmds.getAttr("redshiftOptions.imageFormat") - ] - else: - # Get the extension, getAttr defaultRenderGlobals.imageFormat - # returns an index number. - filename_base = os.path.basename(filename_0) - extension = os.path.splitext(filename_base)[-1].strip(".") - - filename_prefix = cmds.getAttr(prefix_attr) - return {"ext": extension, - "filename_prefix": filename_prefix, - "padding": padding, - "filename_0": filename_0} - - -class MayaSubmitDeadline(pyblish.api.InstancePlugin): +class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): """Submit available render layers to Deadline. Renders are submitted to a Deadline Web Service as @@ -284,15 +63,12 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin): """ - label = "Submit to Deadline" - order = pyblish.api.IntegratorOrder + 0.1 + label = "Submit Render to Deadline" hosts = ["maya"] families = ["renderlayer"] targets = ["local"] - use_published = True tile_assembler_plugin = "OpenPypeTileAssembler" - asset_dependencies = False priority = 50 tile_priority = 50 limit_groups = [] @@ -300,32 +76,173 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin): pluginInfo = {} group = "none" - def process(self, instance): - """Plugin entry point.""" - instance.data["toBeRenderedOn"] = "deadline" + def get_job_info(self): + job_info = DeadlineJobInfo(Plugin="MayaBatch") + + # todo: test whether this works for existing production cases + # where custom jobInfo was stored in the project settings + for key, value in self.jobInfo.items(): + setattr(job_info, key, value) + + instance = self._instance context = instance.context - self._instance = instance - self.payload_skeleton = copy.deepcopy(payload_skeleton_template) + filepath = context.data["currentFile"] + filename = os.path.basename(filepath) - # get default deadline webservice url from deadline module - self.deadline_url = instance.context.data.get("defaultDeadline") - # if custom one is set in instance, use that - if instance.data.get("deadlineUrl"): - self.deadline_url = instance.data.get("deadlineUrl") - assert self.deadline_url, "Requires Deadline Webservice URL" + job_info.Name = "%s - %s" % (filename, instance.name) + job_info.BatchName = filename + job_info.Plugin = instance.data.get("mayaRenderPlugin", "MayaBatch") + job_info.UserName = context.data.get( + "deadlineUser", getpass.getuser()) - # just using existing names from Setting - self._job_info = self.jobInfo + # Deadline requires integers in frame range + frames = "{start}-{end}x{step}".format( + start=int(instance.data["frameStartHandle"]), + end=int(instance.data["frameEndHandle"]), + step=int(instance.data["byFrameStep"]), + ) + job_info.Frames = frames - self._plugin_info = self.pluginInfo + job_info.Pool = instance.data.get("primaryPool") + job_info.SecondaryPool = instance.data.get("secondaryPool") + job_info.ChunkSize = instance.data.get("chunkSize", 10) + job_info.Comment = context.data.get("comment") + job_info.Priority = instance.data.get("priority", self.priority) + + if self.group != "none" and self.group: + job_info.Group = self.group + + if self.limit_groups: + job_info.LimitGroups = ",".join(self.limit_groups) + + self.payload_skeleton["JobInfo"]["Name"] = jobname + self.payload_skeleton["JobInfo"]["BatchName"] = src_filename + + # Optional, enable double-click to preview rendered + # frames from Deadline Monitor + self.payload_skeleton["JobInfo"]["OutputDirectory0"] = \ + os.path.dirname(output_filename_0).replace("\\", "/") + self.payload_skeleton["JobInfo"]["OutputFilename0"] = \ + output_filename_0.replace("\\", "/") + + # Add options from RenderGlobals------------------------------------- + render_globals = instance.data.get("renderGlobals", {}) + self.payload_skeleton["JobInfo"].update(render_globals) + + keys = [ + "FTRACK_API_KEY", + "FTRACK_API_USER", + "FTRACK_SERVER", + "OPENPYPE_SG_USER", + "AVALON_PROJECT", + "AVALON_ASSET", + "AVALON_TASK", + "AVALON_APP_NAME", + "OPENPYPE_DEV", + "OPENPYPE_LOG_NO_COLORS", + "OPENPYPE_VERSION" + ] + # Add mongo url if it's enabled + if self._instance.context.data.get("deadlinePassMongoUrl"): + keys.append("OPENPYPE_MONGO") + + environment = dict({key: os.environ[key] for key in keys + if key in os.environ}, **legacy_io.Session) + + + # TODO: Taken from old publish class - test whether still needed + environment["OPENPYPE_LOG_NO_COLORS"] = "1" + environment["OPENPYPE_MAYA_VERSION"] = cmds.about(v=True) + # to recognize job from PYPE for turning Event On/Off + environment["OPENPYPE_RENDER_JOB"] = "1" + + for key in keys: + val = environment.get(key) + if val: + job_info.EnvironmentKeyValue = "{key}={value}".format( + key=key, + value=val + ) + # to recognize job from PYPE for turning Event On/Off + job_info.EnvironmentKeyValue = "OPENPYPE_RENDER_JOB=1" + + for i, filepath in enumerate(instance.data["files"]): + dirname = os.path.dirname(filepath) + fname = os.path.basename(filepath) + job_info.OutputDirectory = dirname.replace("\\", "/") + job_info.OutputFilename = fname + + # Adding file dependencies. + if self.asset_dependencies: + dependencies = instance.context.data["fileDependencies"] + dependencies.append(context.data["currentFile"]) + for dependency in dependencies: + job_info.AssetDependency = dependency + + # Add list of expected files to job + # --------------------------------- + exp = instance.data.get("expectedFiles") + + def _get_output_filename(files): + col, rem = clique.assemble(files) + if not col and rem: + # we couldn't find any collections but have + # individual files. + assert len(rem) == 1, ( + "Found multiple non related files " + "to render, don't know what to do " + "with them.") + return rem[0] + else: + return col[0].format('{head}{padding}{tail}') + + if isinstance(exp[0], dict): + # we have aovs and we need to iterate over them + for _aov, files in exp[0].items(): + output_file = _get_output_filename(files) + job_info.OutputFilename = output_file + else: + output_file = _get_output_filename(exp) + job_info.OutputFilename = output_file + + return job_info + + def get_plugin_info(self): + + instance = self._instance + context = instance.context + + renderlayer = instance.data['setMembers'] # rs_beauty + + self.payload_skeleton["PluginInfo"]["RenderLayer"] = renderlayer + self.payload_skeleton["PluginInfo"]["RenderSetupIncludeLights"] = instance.data.get("renderSetupIncludeLights") # noqa + + # Output driver to render + plugin_info = DeadlinePluginInfo( + SceneFile=context.data["currentFile"], + Version=cmds.about(version=True), + ) + + return attr.asdict(plugin_info) + + def process_submission(self): + # Override to NOT submit by default when calling super process() method + pass + + def process(self, instance): + super(MayaSubmitDeadline, self).process(instance) + + # TODO: Avoid the need for this logic here, needed for submit publish + # Store output dir for unified publisher (filesequence) + output_dir = os.path.dirname(instance.data["files"][0]) + instance.data["outputDir"] = output_dir + instance.data["toBeRenderedOn"] = "deadline" self.limit_groups = self.limit context = instance.context workspace = context.data["workspaceDir"] - anatomy = context.data['anatomy'] - instance.data["toBeRenderedOn"] = "deadline" filepath = None patches = ( @@ -336,80 +253,24 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin): "scene_patches", {}) ) - # Handle render/export from published scene or not ------------------ - if self.use_published: - patched_files = [] - for i in context: - if "workfile" not in i.data["families"]: - continue - assert i.data["publish"] is True, ( - "Workfile (scene) must be published along") - template_data = i.data.get("anatomyData") - rep = i.data.get("representations")[0].get("name") - template_data["representation"] = rep - template_data["ext"] = rep - template_data["comment"] = None - anatomy_filled = anatomy.format(template_data) - template_filled = anatomy_filled["publish"]["path"] - filepath = os.path.normpath(template_filled) - self.log.info("Using published scene for render {}".format( - filepath)) + # todo: on self.use_published originally use template_data["representation"] using .get("name") instead of .get("ext") + # todo: on self.use_published replace path for publishRenderMetadataFolder + # todo: on self.use_published apply scene patches to workfile instance + # rep = i.data.get("representations")[0].get("name") - if not os.path.exists(filepath): - self.log.error("published scene does not exist!") - raise - # now we need to switch scene in expected files - # because token will now point to published - # scene file and that might differ from current one - new_scene = os.path.splitext( - os.path.basename(filepath))[0] - orig_scene = os.path.splitext( - os.path.basename(context.data["currentFile"]))[0] - exp = instance.data.get("expectedFiles") + # if instance.data.get("publishRenderMetadataFolder"): + # instance.data["publishRenderMetadataFolder"] = \ + # instance.data["publishRenderMetadataFolder"].replace( + # orig_scene, new_scene) + # self.log.info("Scene name was switched {} -> {}".format( + # orig_scene, new_scene + # )) + # # patch workfile is needed + # if filepath not in patched_files: + # patched_file = self._patch_workfile(filepath, patches) + # patched_files.append(patched_file) - if isinstance(exp[0], dict): - # we have aovs and we need to iterate over them - new_exp = {} - for aov, files in exp[0].items(): - replaced_files = [] - for f in files: - replaced_files.append( - f.replace(orig_scene, new_scene) - ) - new_exp[aov] = replaced_files - instance.data["expectedFiles"] = [new_exp] - else: - new_exp = [] - for f in exp: - new_exp.append( - f.replace(orig_scene, new_scene) - ) - instance.data["expectedFiles"] = [new_exp] - - if instance.data.get("publishRenderMetadataFolder"): - instance.data["publishRenderMetadataFolder"] = \ - instance.data["publishRenderMetadataFolder"].replace( - orig_scene, new_scene) - self.log.info("Scene name was switched {} -> {}".format( - orig_scene, new_scene - )) - # patch workfile is needed - if filepath not in patched_files: - patched_file = self._patch_workfile(filepath, patches) - patched_files.append(patched_file) - - all_instances = [] - for result in context.data["results"]: - if (result["instance"] is not None and - result["instance"] not in all_instances): # noqa: E128 - all_instances.append(result["instance"]) - - # fallback if nothing was set - if not filepath: - self.log.warning("Falling back to workfile") - filepath = context.data["currentFile"] - - self.log.debug(filepath) + filepath = self.scene_path # collect by super().process # Gather needed data ------------------------------------------------ default_render_file = instance.context.data.get('project_settings')\ @@ -417,10 +278,8 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin): .get('RenderSettings')\ .get('default_render_image_folder') filename = os.path.basename(filepath) - comment = context.data.get("comment", "") dirname = os.path.join(workspace, default_render_file) renderlayer = instance.data['setMembers'] # rs_beauty - deadline_user = context.data.get("user", getpass.getuser()) # Always use the original work file name for the Job name even when # rendering is done from the published Work File. The original work @@ -454,116 +313,34 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin): pass # Fill in common data to payload ------------------------------------ - payload_data = {} - payload_data["filename"] = filename - payload_data["filepath"] = filepath - payload_data["jobname"] = jobname - payload_data["deadline_user"] = deadline_user - payload_data["comment"] = comment - payload_data["output_filename_0"] = output_filename_0 - payload_data["render_variables"] = render_variables - payload_data["renderlayer"] = renderlayer - payload_data["workspace"] = workspace - payload_data["dirname"] = dirname - - self.log.info("--- Submission data:") - for k, v in payload_data.items(): - self.log.info("- {}: {}".format(k, v)) - self.log.info("-" * 20) - - frame_pattern = self.payload_skeleton["JobInfo"]["Frames"] - self.payload_skeleton["JobInfo"]["Frames"] = frame_pattern.format( - start=int(self._instance.data["frameStartHandle"]), - end=int(self._instance.data["frameEndHandle"]), - step=int(self._instance.data["byFrameStep"])) - - self.payload_skeleton["JobInfo"]["Plugin"] = self._instance.data.get( - "mayaRenderPlugin", "MayaBatch") - - self.payload_skeleton["JobInfo"]["BatchName"] = src_filename - # Job name, as seen in Monitor - self.payload_skeleton["JobInfo"]["Name"] = jobname - # Arbitrary username, for visualisation in Monitor - self.payload_skeleton["JobInfo"]["UserName"] = deadline_user - # Set job priority - self.payload_skeleton["JobInfo"]["Priority"] = \ - self._instance.data.get("priority", self.priority) - - if self.group != "none" and self.group: - self.payload_skeleton["JobInfo"]["Group"] = self.group - - if self.limit_groups: - self.payload_skeleton["JobInfo"]["LimitGroups"] = \ - ",".join(self.limit_groups) - # Optional, enable double-click to preview rendered - # frames from Deadline Monitor - self.payload_skeleton["JobInfo"]["OutputDirectory0"] = \ - os.path.dirname(output_filename_0).replace("\\", "/") - self.payload_skeleton["JobInfo"]["OutputFilename0"] = \ - output_filename_0.replace("\\", "/") - - self.payload_skeleton["JobInfo"]["Comment"] = comment - self.payload_skeleton["PluginInfo"]["RenderLayer"] = renderlayer - - self.payload_skeleton["PluginInfo"]["RenderSetupIncludeLights"] = instance.data.get("renderSetupIncludeLights") # noqa - # Adding file dependencies. - dependencies = instance.context.data["fileDependencies"] - dependencies.append(filepath) - if self.asset_dependencies: - for dependency in dependencies: - key = "AssetDependency" + str(dependencies.index(dependency)) - self.payload_skeleton["JobInfo"][key] = dependency - - # Handle environments ----------------------------------------------- - # We need those to pass them to pype for it to set correct context - keys = [ - "FTRACK_API_KEY", - "FTRACK_API_USER", - "FTRACK_SERVER", - "OPENPYPE_SG_USER", - "AVALON_PROJECT", - "AVALON_ASSET", - "AVALON_TASK", - "AVALON_APP_NAME", - "OPENPYPE_DEV", - "OPENPYPE_LOG_NO_COLORS", - "OPENPYPE_VERSION" - ] - # Add mongo url if it's enabled - if instance.context.data.get("deadlinePassMongoUrl"): - keys.append("OPENPYPE_MONGO") - - environment = dict({key: os.environ[key] for key in keys - if key in os.environ}, **legacy_io.Session) - environment["OPENPYPE_LOG_NO_COLORS"] = "1" - environment["OPENPYPE_MAYA_VERSION"] = cmds.about(v=True) - # to recognize job from PYPE for turning Event On/Off - environment["OPENPYPE_RENDER_JOB"] = "1" - self.payload_skeleton["JobInfo"].update({ - "EnvironmentKeyValue%d" % index: "{key}={value}".format( - key=key, - value=environment[key] - ) for index, key in enumerate(environment) - }) - # Add options from RenderGlobals------------------------------------- - render_globals = instance.data.get("renderGlobals", {}) - self.payload_skeleton["JobInfo"].update(render_globals) + payload_data = { + "filename": filename, + "filepath": filepath, + "jobname": jobname, + "comment": comment, + "output_filename_0": output_filename_0, + "render_variables": render_variables, + "renderlayer": renderlayer, + "workspace": workspace, + "dirname": dirname, + } # Submit preceding export jobs ------------------------------------- export_job = None assert not all(x in instance.data["families"] for x in ['vrayscene', 'assscene']), ( "Vray Scene and Ass Scene options are mutually exclusive") - if "vrayscene" in instance.data["families"]: - export_job = self._submit_export(payload_data, "vray") - if "assscene" in instance.data["families"]: - export_job = self._submit_export(payload_data, "arnold") - - # Prepare main render job ------------------------------------------- if "vrayscene" in instance.data["families"]: + vray_export_payload = self._get_vray_export_payload(payload_data) + export_job = self.submit(vray_export_payload) + payload = self._get_vray_render_payload(payload_data) + elif "assscene" in instance.data["families"]: + ass_export_payload = self._get_arnold_export_payload(payload_data) + export_job = self.submit(ass_export_payload) + payload = self._get_arnold_render_payload(payload_data) else: payload = self._get_maya_payload(payload_data) @@ -572,267 +349,222 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin): if export_job: payload["JobInfo"]["JobDependency0"] = export_job - # Add list of expected files to job --------------------------------- - exp = instance.data.get("expectedFiles") - exp_index = 0 - output_filenames = {} - - if isinstance(exp[0], dict): - # we have aovs and we need to iterate over them - for _aov, files in exp[0].items(): - col, rem = clique.assemble(files) - if not col and rem: - # we couldn't find any collections but have - # individual files. - assert len(rem) == 1, ("Found multiple non related files " - "to render, don't know what to do " - "with them.") - output_file = rem[0] - if not instance.data.get("tileRendering"): - payload['JobInfo']['OutputFilename' + str(exp_index)] = output_file # noqa: E501 - else: - output_file = col[0].format('{head}{padding}{tail}') - if not instance.data.get("tileRendering"): - payload['JobInfo']['OutputFilename' + str(exp_index)] = output_file # noqa: E501 - - output_filenames['OutputFilename' + str(exp_index)] = output_file # noqa: E501 - exp_index += 1 - else: - col, rem = clique.assemble(exp) - if not col and rem: - # we couldn't find any collections but have - # individual files. - assert len(rem) == 1, ("Found multiple non related files " - "to render, don't know what to do " - "with them.") - - output_file = rem[0] - if not instance.data.get("tileRendering"): - payload['JobInfo']['OutputFilename' + str(exp_index)] = output_file # noqa: E501 - else: - output_file = col[0].format('{head}{padding}{tail}') - if not instance.data.get("tileRendering"): - payload['JobInfo']['OutputFilename' + str(exp_index)] = output_file # noqa: E501 - - output_filenames['OutputFilename' + str(exp_index)] = output_file - plugin = payload["JobInfo"]["Plugin"] self.log.info("using render plugin : {}".format(plugin)) # Store output dir for unified publisher (filesequence) instance.data["outputDir"] = os.path.dirname(output_filename_0) - self.preflight_check(instance) - # add jobInfo and pluginInfo variables from Settings - payload["JobInfo"].update(self._job_info) - payload["PluginInfo"].update(self._plugin_info) + payload["JobInfo"].update(self.jobInfo) + payload["PluginInfo"].update(self.pluginInfo) - # Prepare tiles data ------------------------------------------------ if instance.data.get("tileRendering"): - # if we have sequence of files, we need to create tile job for - # every frame + # Prepare tiles data + self._tile_render(instance, payload) + else: + # Submit main render job + self.submit(payload) - payload["JobInfo"]["TileJob"] = True - payload["JobInfo"]["TileJobTilesInX"] = instance.data.get("tilesX") - payload["JobInfo"]["TileJobTilesInY"] = instance.data.get("tilesY") - payload["PluginInfo"]["ImageHeight"] = instance.data.get("resolutionHeight") # noqa: E501 - payload["PluginInfo"]["ImageWidth"] = instance.data.get("resolutionWidth") # noqa: E501 - payload["PluginInfo"]["RegionRendering"] = True + def _tile_render(self, instance, payload): - assembly_payload = { - "AuxFiles": [], - "JobInfo": { - "BatchName": payload["JobInfo"]["BatchName"], - "Frames": 1, - "Name": "{} - Tile Assembly Job".format( - payload["JobInfo"]["Name"]), - "OutputDirectory0": - payload["JobInfo"]["OutputDirectory0"].replace( - "\\", "/"), - "Plugin": self.tile_assembler_plugin, - "MachineLimit": 1 - }, - "PluginInfo": { - "CleanupTiles": 1, - "ErrorOnMissing": True - } + # As collected by super process() + job_info = self.job_info + plugin_info = self.pluginInfo + + # if we have sequence of files, we need to create tile job for + # every frame + + job_info.TileJob = True + job_info.TileJobTilesInX = instance.data.get("tilesX") + job_info.TileJobTilesInY = instance.data.get("tilesY") + + plugin_info["ImageHeight"] = instance.data.get("resolutionHeight") + plugin_info["ImageWidth"] = instance.data.get("resolutionWidth") + plugin_info["RegionRendering"] = True + + assembly_payload = { + "AuxFiles": [], + "JobInfo": { + "BatchName": payload["JobInfo"]["BatchName"], + "Frames": 1, + "Name": "{} - Tile Assembly Job".format( + payload["JobInfo"]["Name"]), + "OutputDirectory0": + payload["JobInfo"]["OutputDirectory0"].replace( + "\\", "/"), + "Plugin": self.tile_assembler_plugin, + "MachineLimit": 1 + }, + "PluginInfo": { + "CleanupTiles": 1, + "ErrorOnMissing": True } - assembly_payload["JobInfo"].update(output_filenames) - assembly_payload["JobInfo"]["Priority"] = self._instance.data.get( - "tile_priority", self.tile_priority) - assembly_payload["JobInfo"]["UserName"] = deadline_user + } + assembly_payload["JobInfo"]["Priority"] = self._instance.data.get( + "tile_priority", self.tile_priority) - frame_payloads = [] - assembly_payloads = [] + frame_payloads = [] + assembly_payloads = [] - R_FRAME_NUMBER = re.compile(r".+\.(?P[0-9]+)\..+") # noqa: N806, E501 - REPL_FRAME_NUMBER = re.compile(r"(.+\.)([0-9]+)(\..+)") # noqa: N806, E501 + R_FRAME_NUMBER = re.compile( + r".+\.(?P[0-9]+)\..+") # noqa: N806, E501 + REPL_FRAME_NUMBER = re.compile( + r"(.+\.)([0-9]+)(\..+)") # noqa: N806, E501 - if isinstance(exp[0], dict): - # we have aovs and we need to iterate over them - # get files from `beauty` - files = exp[0].get("beauty") - # assembly files are used for assembly jobs as we need to put - # together all AOVs - assembly_files = list( - itertools.chain.from_iterable( - [f for _, f in exp[0].items()])) - if not files: - # if beauty doesn't exists, use first aov we found - files = exp[0].get(list(exp[0].keys())[0]) - else: - files = exp - assembly_files = files + exp = instance.data["expectedFiles"] + if isinstance(exp[0], dict): + # we have aovs and we need to iterate over them + # get files from `beauty` + files = exp[0].get("beauty") + # assembly files are used for assembly jobs as we need to put + # together all AOVs + assembly_files = list( + itertools.chain.from_iterable( + [f for _, f in exp[0].items()])) + if not files: + # if beauty doesn't exists, use first aov we found + files = exp[0].get(list(exp[0].keys())[0]) + else: + files = exp + assembly_files = files - frame_jobs = {} + frame_jobs = {} - file_index = 1 - for file in files: - frame = re.search(R_FRAME_NUMBER, file).group("frame") - new_payload = copy.deepcopy(payload) - new_payload["JobInfo"]["Name"] = \ - "{} (Frame {} - {} tiles)".format( - payload["JobInfo"]["Name"], - frame, - instance.data.get("tilesX") * instance.data.get("tilesY") # noqa: E501 + file_index = 1 + for file in files: + frame = re.search(R_FRAME_NUMBER, file).group("frame") + new_payload = copy.deepcopy(payload) + new_payload["JobInfo"]["Name"] = \ + "{} (Frame {} - {} tiles)".format( + payload["JobInfo"]["Name"], + frame, + instance.data.get("tilesX") * instance.data.get("tilesY") + # noqa: E501 ) - self.log.info( - "... preparing job {}".format( - new_payload["JobInfo"]["Name"])) - new_payload["JobInfo"]["TileJobFrame"] = frame + self.log.info( + "... preparing job {}".format( + new_payload["JobInfo"]["Name"])) + new_payload["JobInfo"]["TileJobFrame"] = frame - tiles_data = _format_tiles( + tiles_data = _format_tiles( + file, 0, + instance.data.get("tilesX"), + instance.data.get("tilesY"), + instance.data.get("resolutionWidth"), + instance.data.get("resolutionHeight"), + payload["PluginInfo"]["OutputFilePrefix"] + )[0] + new_payload["JobInfo"].update(tiles_data["JobInfo"]) + new_payload["PluginInfo"].update(tiles_data["PluginInfo"]) + + self.log.info("hashing {} - {}".format(file_index, file)) + job_hash = hashlib.sha256( + ("{}_{}".format(file_index, file)).encode("utf-8")) + frame_jobs[frame] = job_hash.hexdigest() + new_payload["JobInfo"]["ExtraInfo0"] = job_hash.hexdigest() + new_payload["JobInfo"]["ExtraInfo1"] = file + + frame_payloads.append(new_payload) + file_index += 1 + + file_index = 1 + for file in assembly_files: + frame = re.search(R_FRAME_NUMBER, file).group("frame") + new_assembly_payload = copy.deepcopy(assembly_payload) + new_assembly_payload["JobInfo"]["Name"] = \ + "{} (Frame {})".format( + assembly_payload["JobInfo"]["Name"], + frame) + new_assembly_payload["JobInfo"]["OutputFilename0"] = re.sub( + REPL_FRAME_NUMBER, + "\\1{}\\3".format("#" * len(frame)), file) + + new_assembly_payload["PluginInfo"]["Renderer"] = \ + self._instance.data["renderer"] # noqa: E501 + new_assembly_payload["JobInfo"]["ExtraInfo0"] = frame_jobs[ + frame] # noqa: E501 + new_assembly_payload["JobInfo"]["ExtraInfo1"] = file + assembly_payloads.append(new_assembly_payload) + file_index += 1 + + self.log.info( + "Submitting tile job(s) [{}] ...".format(len(frame_payloads))) + + url = "{}/api/jobs".format(self.deadline_url) + tiles_count = instance.data.get("tilesX") * instance.data.get( + "tilesY") # noqa: E501 + + for tile_job in frame_payloads: + response = self.submit(tile_job) + + job_id = response.json()["_id"] + hash = response.json()["Props"]["Ex0"] + + for assembly_job in assembly_payloads: + if assembly_job["JobInfo"]["ExtraInfo0"] == hash: + assembly_job["JobInfo"]["JobDependency0"] = job_id + + for assembly_job in assembly_payloads: + file = assembly_job["JobInfo"]["ExtraInfo1"] + # write assembly job config files + now = datetime.now() + + config_file = os.path.join( + os.path.dirname(output_filename_0), + "{}_config_{}.txt".format( + os.path.splitext(file)[0], + now.strftime("%Y_%m_%d_%H_%M_%S") + ) + ) + + try: + if not os.path.isdir(os.path.dirname(config_file)): + os.makedirs(os.path.dirname(config_file)) + except OSError: + # directory is not available + self.log.warning( + "Path is unreachable: `{}`".format( + os.path.dirname(config_file))) + + # add config file as job auxFile + assembly_job["AuxFiles"] = [config_file] + + with open(config_file, "w") as cf: + print("TileCount={}".format(tiles_count), file=cf) + print("ImageFileName={}".format(file), file=cf) + print("ImageWidth={}".format( + instance.data.get("resolutionWidth")), file=cf) + print("ImageHeight={}".format( + instance.data.get("resolutionHeight")), file=cf) + + tiles = _format_tiles( file, 0, instance.data.get("tilesX"), instance.data.get("tilesY"), instance.data.get("resolutionWidth"), instance.data.get("resolutionHeight"), payload["PluginInfo"]["OutputFilePrefix"] - )[0] - new_payload["JobInfo"].update(tiles_data["JobInfo"]) - new_payload["PluginInfo"].update(tiles_data["PluginInfo"]) + )[1] + sorted(tiles) + for k, v in tiles.items(): + print("{}={}".format(k, v), file=cf) - self.log.info("hashing {} - {}".format(file_index, file)) - job_hash = hashlib.sha256( - ("{}_{}".format(file_index, file)).encode("utf-8")) - frame_jobs[frame] = job_hash.hexdigest() - new_payload["JobInfo"]["ExtraInfo0"] = job_hash.hexdigest() - new_payload["JobInfo"]["ExtraInfo1"] = file + job_idx = 1 + instance.data["assemblySubmissionJobs"] = [] + for ass_job in assembly_payloads: + self.log.info("submitting assembly job {} of {}".format( + job_idx, len(assembly_payloads) + )) + response = self.submit(ass_job) - frame_payloads.append(new_payload) - file_index += 1 + instance.data["assemblySubmissionJobs"].append( + response.json()["_id"]) + job_idx += 1 - file_index = 1 - for file in assembly_files: - frame = re.search(R_FRAME_NUMBER, file).group("frame") - new_assembly_payload = copy.deepcopy(assembly_payload) - new_assembly_payload["JobInfo"]["Name"] = \ - "{} (Frame {})".format( - assembly_payload["JobInfo"]["Name"], - frame) - new_assembly_payload["JobInfo"]["OutputFilename0"] = re.sub( - REPL_FRAME_NUMBER, - "\\1{}\\3".format("#" * len(frame)), file) - - new_assembly_payload["PluginInfo"]["Renderer"] = self._instance.data["renderer"] # noqa: E501 - new_assembly_payload["JobInfo"]["ExtraInfo0"] = frame_jobs[frame] # noqa: E501 - new_assembly_payload["JobInfo"]["ExtraInfo1"] = file - assembly_payloads.append(new_assembly_payload) - file_index += 1 - - self.log.info( - "Submitting tile job(s) [{}] ...".format(len(frame_payloads))) - - url = "{}/api/jobs".format(self.deadline_url) - tiles_count = instance.data.get("tilesX") * instance.data.get("tilesY") # noqa: E501 - - for tile_job in frame_payloads: - response = requests_post(url, json=tile_job) - if not response.ok: - raise Exception(response.text) - - job_id = response.json()["_id"] - hash = response.json()["Props"]["Ex0"] - - for assembly_job in assembly_payloads: - if assembly_job["JobInfo"]["ExtraInfo0"] == hash: - assembly_job["JobInfo"]["JobDependency0"] = job_id - - for assembly_job in assembly_payloads: - file = assembly_job["JobInfo"]["ExtraInfo1"] - # write assembly job config files - now = datetime.now() - - config_file = os.path.join( - os.path.dirname(output_filename_0), - "{}_config_{}.txt".format( - os.path.splitext(file)[0], - now.strftime("%Y_%m_%d_%H_%M_%S") - ) - ) - - try: - if not os.path.isdir(os.path.dirname(config_file)): - os.makedirs(os.path.dirname(config_file)) - except OSError: - # directory is not available - self.log.warning( - "Path is unreachable: `{}`".format( - os.path.dirname(config_file))) - - # add config file as job auxFile - assembly_job["AuxFiles"] = [config_file] - - with open(config_file, "w") as cf: - print("TileCount={}".format(tiles_count), file=cf) - print("ImageFileName={}".format(file), file=cf) - print("ImageWidth={}".format( - instance.data.get("resolutionWidth")), file=cf) - print("ImageHeight={}".format( - instance.data.get("resolutionHeight")), file=cf) - - tiles = _format_tiles( - file, 0, - instance.data.get("tilesX"), - instance.data.get("tilesY"), - instance.data.get("resolutionWidth"), - instance.data.get("resolutionHeight"), - payload["PluginInfo"]["OutputFilePrefix"] - )[1] - sorted(tiles) - for k, v in tiles.items(): - print("{}={}".format(k, v), file=cf) - - job_idx = 1 - instance.data["assemblySubmissionJobs"] = [] - for ass_job in assembly_payloads: - self.log.info("submitting assembly job {} of {}".format( - job_idx, len(assembly_payloads) - )) - self.log.debug(json.dumps(ass_job, indent=4, sort_keys=True)) - response = requests_post(url, json=ass_job) - if not response.ok: - raise Exception(response.text) - - instance.data["assemblySubmissionJobs"].append( - response.json()["_id"]) - job_idx += 1 - - instance.data["jobBatchName"] = payload["JobInfo"]["BatchName"] - self.log.info("Setting batch name on instance: {}".format( - instance.data["jobBatchName"])) - else: - # Submit job to farm -------------------------------------------- - self.log.info("Submitting ...") - self.log.debug(json.dumps(payload, indent=4, sort_keys=True)) - - # E.g. http://192.168.0.1:8082/api/jobs - url = "{}/api/jobs".format(self.deadline_url) - response = requests_post(url, json=payload) - if not response.ok: - raise Exception(response.text) - instance.data["deadlineSubmissionJob"] = response.json() + instance.data["jobBatchName"] = payload["JobInfo"]["BatchName"] + self.log.info("Setting batch name on instance: {}".format( + instance.data["jobBatchName"])) def _get_maya_payload(self, data): payload = copy.deepcopy(self.payload_skeleton) @@ -1045,39 +777,6 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin): payload["PluginInfo"].update(plugin_info) return payload - def _submit_export(self, data, format): - if format == "vray": - payload = self._get_vray_export_payload(data) - self.log.info("Submitting vrscene export job.") - elif format == "arnold": - payload = self._get_arnold_export_payload(data) - self.log.info("Submitting ass export job.") - - url = "{}/api/jobs".format(self.deadline_url) - response = requests_post(url, json=payload) - if not response.ok: - self.log.error("Submition failed!") - self.log.error(response.status_code) - self.log.error(response.content) - self.log.debug(payload) - raise RuntimeError(response.text) - - dependency = response.json() - return dependency["_id"] - - def preflight_check(self, instance): - """Ensure the startFrame, endFrame and byFrameStep are integers.""" - for key in ("frameStartHandle", "frameEndHandle", "byFrameStep"): - value = instance.data[key] - - if int(value) == value: - continue - - self.log.warning( - "%f=%d was rounded off to nearest integer" - % (value, int(value)) - ) - def format_vray_output_filename(self, filename, template, dir=False): """Format the expected output file of the Export job. @@ -1160,3 +859,209 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin): "Applied {} patch to scene.".format( patches[i]["name"])) return file + + +def _format_tiles( + filename, index, tiles_x, tiles_y, + width, height, prefix): + """Generate tile entries for Deadline tile job. + + Returns two dictionaries - one that can be directly used in Deadline + job, second that can be used for Deadline Assembly job configuration + file. + + This will format tile names: + + Example:: + { + "OutputFilename0Tile0": "_tile_1x1_4x4_Main_beauty.1001.exr", + "OutputFilename0Tile1": "_tile_2x1_4x4_Main_beauty.1001.exr" + } + + And add tile prefixes like: + + Example:: + Image prefix is: + `maya///_` + + Result for tile 0 for 4x4 will be: + `maya///_tile_1x1_4x4__` + + Calculating coordinates is tricky as in Job they are defined as top, + left, bottom, right with zero being in top-left corner. But Assembler + configuration file takes tile coordinates as X, Y, Width and Height and + zero is bottom left corner. + + Args: + filename (str): Filename to process as tiles. + index (int): Index of that file if it is sequence. + tiles_x (int): Number of tiles in X. + tiles_y (int): Number if tikes in Y. + width (int): Width resolution of final image. + height (int): Height resolution of final image. + prefix (str): Image prefix. + + Returns: + (dict, dict): Tuple of two dictionaires - first can be used to + extend JobInfo, second has tiles x, y, width and height + used for assembler configuration. + + """ + tile = 0 + out = {"JobInfo": {}, "PluginInfo": {}} + cfg = OrderedDict() + w_space = width / tiles_x + h_space = height / tiles_y + + cfg["TilesCropped"] = "False" + + for tile_x in range(1, tiles_x + 1): + for tile_y in reversed(range(1, tiles_y + 1)): + tile_prefix = "_tile_{}x{}_{}x{}_".format( + tile_x, tile_y, + tiles_x, + tiles_y + ) + out_tile_index = "OutputFilename{}Tile{}".format( + str(index), tile + ) + new_filename = "{}/{}{}".format( + os.path.dirname(filename), + tile_prefix, + os.path.basename(filename) + ) + out["JobInfo"][out_tile_index] = new_filename + out["PluginInfo"]["RegionPrefix{}".format(str(tile))] = \ + "/{}".format(tile_prefix).join(prefix.rsplit("/", 1)) + + out["PluginInfo"]["RegionTop{}".format(tile)] = int(height) - (tile_y * h_space) # noqa: E501 + out["PluginInfo"]["RegionBottom{}".format(tile)] = int(height) - ((tile_y - 1) * h_space) - 1 # noqa: E501 + out["PluginInfo"]["RegionLeft{}".format(tile)] = (tile_x - 1) * w_space # noqa: E501 + out["PluginInfo"]["RegionRight{}".format(tile)] = (tile_x * w_space) - 1 # noqa: E501 + + cfg["Tile{}".format(tile)] = new_filename + cfg["Tile{}Tile".format(tile)] = new_filename + cfg["Tile{}FileName".format(tile)] = new_filename + cfg["Tile{}X".format(tile)] = (tile_x - 1) * w_space + + cfg["Tile{}Y".format(tile)] = int(height) - (tile_y * h_space) + + cfg["Tile{}Width".format(tile)] = w_space + cfg["Tile{}Height".format(tile)] = h_space + + tile += 1 + return out, cfg + + +def get_renderer_variables(renderlayer, root): + """Retrieve the extension which has been set in the VRay settings. + + Will return None if the current renderer is not VRay + For Maya 2016.5 and up the renderSetup creates renderSetupLayer node which + start with `rs`. Use the actual node name, do NOT use the `nice name` + + Args: + renderlayer (str): the node name of the renderlayer. + root (str): base path to render + + Returns: + dict + + """ + renderer = lib.get_renderer(renderlayer or lib.get_current_renderlayer()) + render_attrs = lib.RENDER_ATTRS.get(renderer, lib.RENDER_ATTRS["default"]) + + padding = cmds.getAttr("{}.{}".format(render_attrs["node"], + render_attrs["padding"])) + + filename_0 = cmds.renderSettings( + fullPath=True, + gin="#" * int(padding), + lut=True, + layer=renderlayer or lib.get_current_renderlayer())[0] + filename_0 = re.sub('_', '_beauty', + filename_0, flags=re.IGNORECASE) + prefix_attr = "defaultRenderGlobals.imageFilePrefix" + + scene = cmds.file(query=True, sceneName=True) + scene, _ = os.path.splitext(os.path.basename(scene)) + + if renderer == "vray": + renderlayer = renderlayer.split("_")[-1] + # Maya's renderSettings function does not return V-Ray file extension + # so we get the extension from vraySettings + extension = cmds.getAttr("vraySettings.imageFormatStr") + + # When V-Ray image format has not been switched once from default .png + # the getAttr command above returns None. As such we explicitly set + # it to `.png` + if extension is None: + extension = "png" + + if extension in ["exr (multichannel)", "exr (deep)"]: + extension = "exr" + + prefix_attr = "vraySettings.fileNamePrefix" + filename_prefix = cmds.getAttr(prefix_attr) + # we need to determine path for vray as maya `renderSettings` query + # does not work for vray. + + filename_0 = re.sub('', scene, filename_prefix, flags=re.IGNORECASE) # noqa: E501 + filename_0 = re.sub('', renderlayer, filename_0, flags=re.IGNORECASE) # noqa: E501 + filename_0 = "{}.{}.{}".format( + filename_0, "#" * int(padding), extension) + filename_0 = os.path.normpath(os.path.join(root, filename_0)) + elif renderer == "renderman": + prefix_attr = "rmanGlobals.imageFileFormat" + # NOTE: This is guessing extensions from renderman display types. + # Some of them are just framebuffers, d_texture format can be + # set in display setting. We set those now to None, but it + # should be handled more gracefully. + display_types = { + "d_deepexr": "exr", + "d_it": None, + "d_null": None, + "d_openexr": "exr", + "d_png": "png", + "d_pointcloud": "ptc", + "d_targa": "tga", + "d_texture": None, + "d_tiff": "tif" + } + + extension = display_types.get( + cmds.listConnections("rmanDefaultDisplay.displayType")[0], + "exr" + ) or "exr" + + filename_prefix = "{}/{}".format( + cmds.getAttr("rmanGlobals.imageOutputDir"), + cmds.getAttr("rmanGlobals.imageFileFormat") + ) + + renderlayer = renderlayer.split("_")[-1] + + filename_0 = re.sub('', scene, filename_prefix, flags=re.IGNORECASE) # noqa: E501 + filename_0 = re.sub('', renderlayer, filename_0, flags=re.IGNORECASE) # noqa: E501 + filename_0 = re.sub('', "#" * int(padding), filename_0, flags=re.IGNORECASE) # noqa: E501 + filename_0 = re.sub('', extension, filename_0, flags=re.IGNORECASE) # noqa: E501 + filename_0 = os.path.normpath(os.path.join(root, filename_0)) + elif renderer == "redshift": + # mapping redshift extension dropdown values to strings + ext_mapping = ["iff", "exr", "tif", "png", "tga", "jpg"] + extension = ext_mapping[ + cmds.getAttr("redshiftOptions.imageFormat") + ] + else: + # Get the extension, getAttr defaultRenderGlobals.imageFormat + # returns an index number. + filename_base = os.path.basename(filename_0) + extension = os.path.splitext(filename_base)[-1].strip(".") + + filename_prefix = cmds.getAttr(prefix_attr) + return {"ext": extension, + "filename_prefix": filename_prefix, + "padding": padding, + "filename_0": filename_0} + + From aaa95efb3c80080166de20c9f48bbfe6a8c11548 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 24 Aug 2022 13:45:53 +0200 Subject: [PATCH 014/102] Removed submodule vendor/configs/OpenColorIO-Configs --- vendor/configs/OpenColorIO-Configs | 1 - 1 file changed, 1 deletion(-) delete mode 160000 vendor/configs/OpenColorIO-Configs diff --git a/vendor/configs/OpenColorIO-Configs b/vendor/configs/OpenColorIO-Configs deleted file mode 160000 index 0bb079c08b..0000000000 --- a/vendor/configs/OpenColorIO-Configs +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 0bb079c08be410030669cbf5f19ff869b88af953 From 4c16b8930fd38d6c9f792c5983d6e2e88f1587ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Je=C5=BEek?= Date: Wed, 24 Aug 2022 14:14:47 +0200 Subject: [PATCH 015/102] Update openpype/plugins/publish/collect_audio.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- openpype/plugins/publish/collect_audio.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/openpype/plugins/publish/collect_audio.py b/openpype/plugins/publish/collect_audio.py index e2fb766ec4..0825c281ad 100644 --- a/openpype/plugins/publish/collect_audio.py +++ b/openpype/plugins/publish/collect_audio.py @@ -66,7 +66,10 @@ class CollectAudio(pyblish.api.InstancePlugin): self.log.info("Audio Data added to instance ...") def _get_repre_doc(self, instance): - cache = instance.context.data.get("__cache_asset_audio", {}) + cache = instance.context.data.get("__cache_asset_audio") + if cache is None: + cache = {} + instance.context.data["__cache_asset_audio"] = cache asset_name = instance.data["asset"] # first try to get it from cache From 6ff16a6d25f2b81f5634d526059734225c556ede Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Je=C5=BEek?= Date: Wed, 24 Aug 2022 14:14:58 +0200 Subject: [PATCH 016/102] Update openpype/plugins/publish/collect_audio.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- openpype/plugins/publish/collect_audio.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openpype/plugins/publish/collect_audio.py b/openpype/plugins/publish/collect_audio.py index 0825c281ad..7d53b24e54 100644 --- a/openpype/plugins/publish/collect_audio.py +++ b/openpype/plugins/publish/collect_audio.py @@ -101,6 +101,5 @@ class CollectAudio(pyblish.api.InstancePlugin): # update cache cache[asset_name] = repre_doc - instance.context.data["__cache_asset_audio"].update(cache) return repre_doc From da5353aa2dd1d9e774a72dbc57a6ac5d7368afdd Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 24 Aug 2022 14:20:35 +0200 Subject: [PATCH 017/102] git: update gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index ea5b20eb69..4b773e97ed 100644 --- a/.gitignore +++ b/.gitignore @@ -107,3 +107,6 @@ website/.docusaurus mypy.ini tools/run_eventserver.* + +# Developer tools +tools/dev_* From 0f95f87d773ddcbe979fe28d5f0196f1befad38e Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Wed, 24 Aug 2022 14:59:14 +0200 Subject: [PATCH 018/102] More draft refactoring - still not functional (WIP commit for my own sanity) --- .../plugins/publish/submit_maya_deadline.py | 185 ++++-------------- 1 file changed, 35 insertions(+), 150 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 6dfa48a9f8..5a7d0b98c6 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -31,7 +31,6 @@ import clique from maya import cmds -from openpype.hosts.maya.api import lib from openpype.pipeline import legacy_io from openpype_modules.deadline import abstract_submit_deadline @@ -87,11 +86,15 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): instance = self._instance context = instance.context - filepath = context.data["currentFile"] - filename = os.path.basename(filepath) + # Always use the original work file name for the Job name even when + # rendering is done from the published Work File. The original work + # file name is clearer because it can also have subversion strings, + # etc. which are stripped for the published file. + src_filepath = context.data["currentFile"] + src_filename = os.path.basename(src_filepath) - job_info.Name = "%s - %s" % (filename, instance.name) - job_info.BatchName = filename + job_info.Name = "%s - %s" % (src_filename, instance.name) + job_info.BatchName = src_filename job_info.Plugin = instance.data.get("mayaRenderPlugin", "MayaBatch") job_info.UserName = context.data.get( "deadlineUser", getpass.getuser()) @@ -116,9 +119,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): if self.limit_groups: job_info.LimitGroups = ",".join(self.limit_groups) - self.payload_skeleton["JobInfo"]["Name"] = jobname - self.payload_skeleton["JobInfo"]["BatchName"] = src_filename - # Optional, enable double-click to preview rendered # frames from Deadline Monitor self.payload_skeleton["JobInfo"]["OutputDirectory0"] = \ @@ -227,11 +227,17 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): return attr.asdict(plugin_info) def process_submission(self): - # Override to NOT submit by default when calling super process() method - pass - def process(self, instance): - super(MayaSubmitDeadline, self).process(instance) + instance = self._instance + context = instance.context + + # Generated by AbstractSubmitDeadline. The `job_info`, `plugin_info` + # and `aux_files` are the skeleton payloads that are the basis for + # all the maya submissions + job_info = self.job_info + plugin_info = self.plugin_info + aux_files = self.aux_files + filepath = self.scene_path # publish if `use_publish` else workfile # TODO: Avoid the need for this logic here, needed for submit publish # Store output dir for unified publisher (filesequence) @@ -241,21 +247,19 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): self.limit_groups = self.limit - context = instance.context - workspace = context.data["workspaceDir"] - - filepath = None - patches = ( - context.data["project_settings"].get( - "deadline", {}).get( - "publish", {}).get( - "MayaSubmitDeadline", {}).get( - "scene_patches", {}) - ) + # Patch workfile (only when use_published is enabled) + if self.use_published: + patches = ( + context.data["project_settings"].get( + "deadline", {}).get( + "publish", {}).get( + "MayaSubmitDeadline", {}).get( + "scene_patches", {}) + ) + self._patch_workfile(filepath, patches) # todo: on self.use_published originally use template_data["representation"] using .get("name") instead of .get("ext") # todo: on self.use_published replace path for publishRenderMetadataFolder - # todo: on self.use_published apply scene patches to workfile instance # rep = i.data.get("representations")[0].get("name") # if instance.data.get("publishRenderMetadataFolder"): @@ -270,9 +274,8 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): # patched_file = self._patch_workfile(filepath, patches) # patched_files.append(patched_file) - filepath = self.scene_path # collect by super().process - # Gather needed data ------------------------------------------------ + workspace = context.data["workspaceDir"] default_render_file = instance.context.data.get('project_settings')\ .get('maya')\ .get('RenderSettings')\ @@ -281,14 +284,10 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): dirname = os.path.join(workspace, default_render_file) renderlayer = instance.data['setMembers'] # rs_beauty - # Always use the original work file name for the Job name even when - # rendering is done from the published Work File. The original work - # file name is clearer because it can also have subversion strings, - # etc. which are stripped for the published file. - src_filename = os.path.basename(context.data["currentFile"]) - jobname = "%s - %s" % (src_filename, instance.name) - # Get the variables depending on the renderer + # TODO: Find replacement logic for `get_renderer_variables` through + # what is collected for the render or is implemented in maya + # api `lib_renderproducts` render_variables = get_renderer_variables(renderlayer, dirname) filename_0 = render_variables["filename_0"] if self.use_published: @@ -842,8 +841,8 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): str: Patched file path or None """ - if os.path.splitext(file)[1].lower() != ".ma" or not patches: - return None + if not patches or os.path.splitext(file)[1].lower() != ".ma": + return compiled_regex = [re.compile(p["regex"]) for p in patches] with open(file, "r+") as pf: @@ -931,7 +930,7 @@ def _format_tiles( os.path.basename(filename) ) out["JobInfo"][out_tile_index] = new_filename - out["PluginInfo"]["RegionPrefix{}".format(str(tile))] = \ + out["PluginInfo"]["RegionPrefix{}".format(tile)] = \ "/{}".format(tile_prefix).join(prefix.rsplit("/", 1)) out["PluginInfo"]["RegionTop{}".format(tile)] = int(height) - (tile_y * h_space) # noqa: E501 @@ -951,117 +950,3 @@ def _format_tiles( tile += 1 return out, cfg - - -def get_renderer_variables(renderlayer, root): - """Retrieve the extension which has been set in the VRay settings. - - Will return None if the current renderer is not VRay - For Maya 2016.5 and up the renderSetup creates renderSetupLayer node which - start with `rs`. Use the actual node name, do NOT use the `nice name` - - Args: - renderlayer (str): the node name of the renderlayer. - root (str): base path to render - - Returns: - dict - - """ - renderer = lib.get_renderer(renderlayer or lib.get_current_renderlayer()) - render_attrs = lib.RENDER_ATTRS.get(renderer, lib.RENDER_ATTRS["default"]) - - padding = cmds.getAttr("{}.{}".format(render_attrs["node"], - render_attrs["padding"])) - - filename_0 = cmds.renderSettings( - fullPath=True, - gin="#" * int(padding), - lut=True, - layer=renderlayer or lib.get_current_renderlayer())[0] - filename_0 = re.sub('_', '_beauty', - filename_0, flags=re.IGNORECASE) - prefix_attr = "defaultRenderGlobals.imageFilePrefix" - - scene = cmds.file(query=True, sceneName=True) - scene, _ = os.path.splitext(os.path.basename(scene)) - - if renderer == "vray": - renderlayer = renderlayer.split("_")[-1] - # Maya's renderSettings function does not return V-Ray file extension - # so we get the extension from vraySettings - extension = cmds.getAttr("vraySettings.imageFormatStr") - - # When V-Ray image format has not been switched once from default .png - # the getAttr command above returns None. As such we explicitly set - # it to `.png` - if extension is None: - extension = "png" - - if extension in ["exr (multichannel)", "exr (deep)"]: - extension = "exr" - - prefix_attr = "vraySettings.fileNamePrefix" - filename_prefix = cmds.getAttr(prefix_attr) - # we need to determine path for vray as maya `renderSettings` query - # does not work for vray. - - filename_0 = re.sub('', scene, filename_prefix, flags=re.IGNORECASE) # noqa: E501 - filename_0 = re.sub('', renderlayer, filename_0, flags=re.IGNORECASE) # noqa: E501 - filename_0 = "{}.{}.{}".format( - filename_0, "#" * int(padding), extension) - filename_0 = os.path.normpath(os.path.join(root, filename_0)) - elif renderer == "renderman": - prefix_attr = "rmanGlobals.imageFileFormat" - # NOTE: This is guessing extensions from renderman display types. - # Some of them are just framebuffers, d_texture format can be - # set in display setting. We set those now to None, but it - # should be handled more gracefully. - display_types = { - "d_deepexr": "exr", - "d_it": None, - "d_null": None, - "d_openexr": "exr", - "d_png": "png", - "d_pointcloud": "ptc", - "d_targa": "tga", - "d_texture": None, - "d_tiff": "tif" - } - - extension = display_types.get( - cmds.listConnections("rmanDefaultDisplay.displayType")[0], - "exr" - ) or "exr" - - filename_prefix = "{}/{}".format( - cmds.getAttr("rmanGlobals.imageOutputDir"), - cmds.getAttr("rmanGlobals.imageFileFormat") - ) - - renderlayer = renderlayer.split("_")[-1] - - filename_0 = re.sub('', scene, filename_prefix, flags=re.IGNORECASE) # noqa: E501 - filename_0 = re.sub('', renderlayer, filename_0, flags=re.IGNORECASE) # noqa: E501 - filename_0 = re.sub('', "#" * int(padding), filename_0, flags=re.IGNORECASE) # noqa: E501 - filename_0 = re.sub('', extension, filename_0, flags=re.IGNORECASE) # noqa: E501 - filename_0 = os.path.normpath(os.path.join(root, filename_0)) - elif renderer == "redshift": - # mapping redshift extension dropdown values to strings - ext_mapping = ["iff", "exr", "tif", "png", "tga", "jpg"] - extension = ext_mapping[ - cmds.getAttr("redshiftOptions.imageFormat") - ] - else: - # Get the extension, getAttr defaultRenderGlobals.imageFormat - # returns an index number. - filename_base = os.path.basename(filename_0) - extension = os.path.splitext(filename_base)[-1].strip(".") - - filename_prefix = cmds.getAttr(prefix_attr) - return {"ext": extension, - "filename_prefix": filename_prefix, - "padding": padding, - "filename_0": filename_0} - - From 67fd21edae5d7c3670ee704558ff10505c64a783 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 15:40:32 +0200 Subject: [PATCH 019/102] imlemented fusion addon --- openpype/hosts/fusion/__init__.py | 10 ++++++++++ openpype/hosts/fusion/addon.py | 23 +++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 openpype/hosts/fusion/addon.py diff --git a/openpype/hosts/fusion/__init__.py b/openpype/hosts/fusion/__init__.py index e69de29bb2..ddae01890b 100644 --- a/openpype/hosts/fusion/__init__.py +++ b/openpype/hosts/fusion/__init__.py @@ -0,0 +1,10 @@ +from .addon import ( + FusionAddon, + FUSION_HOST_DIR, +) + + +__all__ = ( + "FusionAddon", + "FUSION_HOST_DIR", +) diff --git a/openpype/hosts/fusion/addon.py b/openpype/hosts/fusion/addon.py new file mode 100644 index 0000000000..97fb262517 --- /dev/null +++ b/openpype/hosts/fusion/addon.py @@ -0,0 +1,23 @@ +import os +from openpype.modules import OpenPypeModule +from openpype.modules.interfaces import IHostModule + +FUSION_HOST_DIR = os.path.dirname(os.path.abspath(__file__)) + + +class FusionAddon(OpenPypeModule, IHostModule): + name = "fusion" + host_name = "fusion" + + def initialize(self, module_settings): + self.enabled = True + + def get_launch_hook_paths(self, app): + if app.host_name != self.host_name: + return [] + return [ + os.path.join(FUSION_HOST_DIR, "hooks") + ] + + def get_workfile_extensions(self): + return [".comp"] From 3ea7510693ec23623ec7760f3479e10b363a2adf Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 15:40:40 +0200 Subject: [PATCH 020/102] removed usage of HOST_WORKFILE_EXTENSIONS --- openpype/hosts/fusion/api/workio.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/openpype/hosts/fusion/api/workio.py b/openpype/hosts/fusion/api/workio.py index a1710c6e3a..89752d3e6d 100644 --- a/openpype/hosts/fusion/api/workio.py +++ b/openpype/hosts/fusion/api/workio.py @@ -2,13 +2,11 @@ import sys import os -from openpype.pipeline import HOST_WORKFILE_EXTENSIONS - from .pipeline import get_current_comp def file_extensions(): - return HOST_WORKFILE_EXTENSIONS["fusion"] + return [".comp"] def has_unsaved_changes(): From cf50d1dd1f00fe178d8bfc28a039579222cab1d6 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 15:40:56 +0200 Subject: [PATCH 021/102] reuse 'FUSION_HOST_DIR' from fusion public api --- openpype/hosts/fusion/api/pipeline.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/openpype/hosts/fusion/api/pipeline.py b/openpype/hosts/fusion/api/pipeline.py index 54a6c94b60..987eae214b 100644 --- a/openpype/hosts/fusion/api/pipeline.py +++ b/openpype/hosts/fusion/api/pipeline.py @@ -18,12 +18,11 @@ from openpype.pipeline import ( deregister_inventory_action_path, AVALON_CONTAINER_ID, ) -import openpype.hosts.fusion +from openpype.hosts.fusion import FUSION_HOST_DIR log = Logger.get_logger(__name__) -HOST_DIR = os.path.dirname(os.path.abspath(openpype.hosts.fusion.__file__)) -PLUGINS_DIR = os.path.join(HOST_DIR, "plugins") +PLUGINS_DIR = os.path.join(FUSION_HOST_DIR, "plugins") PUBLISH_PATH = os.path.join(PLUGINS_DIR, "publish") LOAD_PATH = os.path.join(PLUGINS_DIR, "load") From 531682f316bc0ec3212ac31edc2881a988385056 Mon Sep 17 00:00:00 2001 From: Felix David Date: Thu, 25 Aug 2022 17:17:56 +0200 Subject: [PATCH 022/102] Kitsu|Change: Drop 'entities root' setting. Closes #3738 --- .../modules/kitsu/utils/update_op_with_zou.py | 109 +++++------------- .../defaults/project_settings/kitsu.json | 4 - .../projects_schema/schema_project_kitsu.json | 17 --- 3 files changed, 28 insertions(+), 102 deletions(-) diff --git a/openpype/modules/kitsu/utils/update_op_with_zou.py b/openpype/modules/kitsu/utils/update_op_with_zou.py index e03cf2b30e..7c97e126e5 100644 --- a/openpype/modules/kitsu/utils/update_op_with_zou.py +++ b/openpype/modules/kitsu/utils/update_op_with_zou.py @@ -166,50 +166,21 @@ def update_op_assets( # Substitute item type for general classification (assets or shots) if item_type in ["Asset", "AssetType"]: - substitute_item_type = "assets" + entity_root_asset_name = "Assets" elif item_type in ["Episode", "Sequence"]: - substitute_item_type = "shots" - else: - substitute_item_type = f"{item_type.lower()}s" - entity_parent_folders = [ - f - for f in project_module_settings["entities_root"] - .get(substitute_item_type) - .split("/") - if f - ] + entity_root_asset_name = "Shots" # Root parent folder if exist visual_parent_doc_id = ( asset_doc_ids[parent_zou_id]["_id"] if parent_zou_id else None ) if visual_parent_doc_id is None: - # Find root folder docs - root_folder_docs = get_assets( + # Find root folder doc ("Assets" or "Shots") + root_folder_doc = get_asset_by_name( project_name, - asset_names=[entity_parent_folders[-1]], + asset_name=entity_root_asset_name, fields=["_id", "data.root_of"], ) - # NOTE: Not sure why it's checking for entity type? - # OP3 does not support multiple assets with same names so type - # filtering is irelevant. - # This way mimics previous implementation: - # ``` - # root_folder_doc = dbcon.find_one( - # { - # "type": "asset", - # "name": entity_parent_folders[-1], - # "data.root_of": substitute_item_type, - # }, - # ["_id"], - # ) - # ``` - root_folder_doc = None - for folder_doc in root_folder_docs: - root_of = folder_doc.get("data", {}).get("root_of") - if root_of == substitute_item_type: - root_folder_doc = folder_doc - break if root_folder_doc: visual_parent_doc_id = root_folder_doc["_id"] @@ -240,7 +211,7 @@ def update_op_assets( item_name = item["name"] # Set root folders parents - item_data["parents"] = entity_parent_folders + item_data["parents"] + item_data["parents"] = [entity_root_asset_name] + item_data["parents"] # Update 'data' different in zou DB updated_data = { @@ -396,54 +367,30 @@ def sync_project_from_kitsu(dbcon: AvalonMongoDB, project: dict): zou_ids_and_asset_docs[project["id"]] = project_doc # Create entities root folders - project_module_settings = get_project_settings(project_name)["kitsu"] - for entity_type, root in project_module_settings["entities_root"].items(): - parent_folders = root.split("/") - direct_parent_doc = None - for i, folder in enumerate(parent_folders, 1): - parent_doc = get_asset_by_name( - project_name, folder, fields=["_id", "data.root_of"] - ) - # NOTE: Not sure why it's checking for entity type? - # OP3 does not support multiple assets with same names so type - # filtering is irelevant. - # Also all of the entities could find be queried at once using - # 'get_assets'. - # This way mimics previous implementation: - # ``` - # parent_doc = dbcon.find_one( - # {"type": "asset", "name": folder, "data.root_of": entity_type} - # ) - # ``` - if ( - parent_doc - and parent_doc.get("data", {}).get("root_of") != entity_type - ): - parent_doc = None - - if not parent_doc: - direct_parent_doc = dbcon.insert_one( - { - "name": folder, - "type": "asset", - "schema": "openpype:asset-3.0", - "data": { - "root_of": entity_type, - "parents": parent_folders[:i], - "visualParent": direct_parent_doc.inserted_id - if direct_parent_doc - else None, - "tasks": {}, - }, - } - ) + to_insert = [ + { + "name": r, + "type": "asset", + "schema": "openpype:asset-3.0", + "data": { + "root_of": r, + "tasks": {}, + }, + } + for r in ["Assets", "Shots"] + if not get_asset_by_name( + project_name, r, fields=["_id", "data.root_of"] + ) + ] # Create - to_insert = [ - create_op_asset(item) - for item in all_entities - if item["id"] not in zou_ids_and_asset_docs.keys() - ] + to_insert.extend( + [ + create_op_asset(item) + for item in all_entities + if item["id"] not in zou_ids_and_asset_docs.keys() + ] + ) if to_insert: # Insert doc in DB dbcon.insert_many(to_insert) diff --git a/openpype/settings/defaults/project_settings/kitsu.json b/openpype/settings/defaults/project_settings/kitsu.json index ba02d8d259..3a9723b9c0 100644 --- a/openpype/settings/defaults/project_settings/kitsu.json +++ b/openpype/settings/defaults/project_settings/kitsu.json @@ -1,8 +1,4 @@ { - "entities_root": { - "assets": "Assets", - "shots": "Shots" - }, "entities_naming_pattern": { "episode": "E##", "sequence": "SQ##", diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_kitsu.json b/openpype/settings/entities/schemas/projects_schema/schema_project_kitsu.json index 014a1b7886..fb47670e74 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_kitsu.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_kitsu.json @@ -5,23 +5,6 @@ "collapsible": true, "is_file": true, "children": [ - { - "type": "dict", - "key": "entities_root", - "label": "Entities root folder", - "children": [ - { - "type": "text", - "key": "assets", - "label": "Assets:" - }, - { - "type": "text", - "key": "shots", - "label": "Shots (includes Episodes & Sequences if any):" - } - ] - }, { "type": "dict", "key": "entities_naming_pattern", From 38932ba3012668e8ca2a239ffa2b68b002d979d6 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Sat, 27 Aug 2022 10:38:01 +0200 Subject: [PATCH 023/102] fixed interface name --- openpype/hosts/fusion/addon.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/fusion/addon.py b/openpype/hosts/fusion/addon.py index 97fb262517..e257005061 100644 --- a/openpype/hosts/fusion/addon.py +++ b/openpype/hosts/fusion/addon.py @@ -1,11 +1,11 @@ import os from openpype.modules import OpenPypeModule -from openpype.modules.interfaces import IHostModule +from openpype.modules.interfaces import IHostAddon FUSION_HOST_DIR = os.path.dirname(os.path.abspath(__file__)) -class FusionAddon(OpenPypeModule, IHostModule): +class FusionAddon(OpenPypeModule, IHostAddon): name = "fusion" host_name = "fusion" From dc4c32b6fc6aaeaedb6bb9d76c7b72d9f5c45c45 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 29 Aug 2022 19:14:58 +0200 Subject: [PATCH 024/102] Fix representation data for workfile --- openpype/modules/deadline/abstract_submit_deadline.py | 6 +++--- .../deadline/plugins/publish/submit_maya_deadline.py | 3 --- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py index 577378335e..d198542370 100644 --- a/openpype/modules/deadline/abstract_submit_deadline.py +++ b/openpype/modules/deadline/abstract_submit_deadline.py @@ -554,9 +554,9 @@ class AbstractSubmitDeadline(pyblish.api.InstancePlugin): "Workfile (scene) must be published along") # determine published path from Anatomy. template_data = i.data.get("anatomyData") - rep = i.data.get("representations")[0].get("ext") - template_data["representation"] = rep - template_data["ext"] = rep + rep = i.data.get("representations")[0] + template_data["representation"] = rep.get("name") + template_data["ext"] = rep.get("ext") template_data["comment"] = None anatomy_filled = anatomy.format(template_data) template_filled = anatomy_filled["publish"]["path"] diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 5a7d0b98c6..68e8eaaa73 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -258,10 +258,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): ) self._patch_workfile(filepath, patches) - # todo: on self.use_published originally use template_data["representation"] using .get("name") instead of .get("ext") # todo: on self.use_published replace path for publishRenderMetadataFolder - # rep = i.data.get("representations")[0].get("name") - # if instance.data.get("publishRenderMetadataFolder"): # instance.data["publishRenderMetadataFolder"] = \ # instance.data["publishRenderMetadataFolder"].replace( From 67b8664be07fbb8a0061b7c8a62bf4073ef79307 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 29 Aug 2022 19:15:20 +0200 Subject: [PATCH 025/102] Remove comment for patched file code refactor since it's already implemented --- .../modules/deadline/plugins/publish/submit_maya_deadline.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 68e8eaaa73..07ed237c94 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -266,10 +266,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): # self.log.info("Scene name was switched {} -> {}".format( # orig_scene, new_scene # )) - # # patch workfile is needed - # if filepath not in patched_files: - # patched_file = self._patch_workfile(filepath, patches) - # patched_files.append(patched_file) # Gather needed data ------------------------------------------------ workspace = context.data["workspaceDir"] From 2da8f036dee501be62da07d07fa9efafc9e8839f Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 29 Aug 2022 19:19:37 +0200 Subject: [PATCH 026/102] Refactor logic for less indentation --- .../deadline/abstract_submit_deadline.py | 103 +++++++++--------- 1 file changed, 52 insertions(+), 51 deletions(-) diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py index d198542370..55e16d8d21 100644 --- a/openpype/modules/deadline/abstract_submit_deadline.py +++ b/openpype/modules/deadline/abstract_submit_deadline.py @@ -546,65 +546,66 @@ class AbstractSubmitDeadline(pyblish.api.InstancePlugin): anatomy = self._instance.context.data['anatomy'] file_path = None for i in self._instance.context: - if "workfile" in i.data["families"] \ - or i.data["family"] == "workfile": - # test if there is instance of workfile waiting - # to be published. - assert i.data["publish"] is True, ( - "Workfile (scene) must be published along") - # determine published path from Anatomy. - template_data = i.data.get("anatomyData") - rep = i.data.get("representations")[0] - template_data["representation"] = rep.get("name") - template_data["ext"] = rep.get("ext") - template_data["comment"] = None - anatomy_filled = anatomy.format(template_data) - template_filled = anatomy_filled["publish"]["path"] - file_path = os.path.normpath(template_filled) - self.log.info("Using published scene for render {}".format( - file_path)) + is_workfile = + if not is_workfile: + continue - if not os.path.exists(file_path): - self.log.error("published scene does not exist!") - raise + # test if there is instance of workfile waiting + # to be published. + assert i.data["publish"] is True, ( + "Workfile (scene) must be published along") + # determine published path from Anatomy. + template_data = i.data.get("anatomyData") + rep = i.data.get("representations")[0] + template_data["representation"] = rep.get("name") + template_data["ext"] = rep.get("ext") + template_data["comment"] = None + anatomy_filled = anatomy.format(template_data) + template_filled = anatomy_filled["publish"]["path"] + file_path = os.path.normpath(template_filled) - if not replace_in_path: - return file_path + self.log.info("Using published scene for render {}".format( + file_path)) - # now we need to switch scene in expected files - # because token will now point to published - # scene file and that might differ from current one - new_scene = os.path.splitext( - os.path.basename(file_path))[0] - orig_scene = os.path.splitext( - os.path.basename( - self._instance.context.data["currentFile"]))[0] - exp = self._instance.data.get("expectedFiles") + if not os.path.exists(file_path): + self.log.error("published scene does not exist!") + raise - if isinstance(exp[0], dict): - # we have aovs and we need to iterate over them - new_exp = {} - for aov, files in exp[0].items(): - replaced_files = [] - for f in files: - replaced_files.append( - str(f).replace(orig_scene, new_scene) - ) - new_exp[aov] = replaced_files - # [] might be too much here, TODO - self._instance.data["expectedFiles"] = [new_exp] - else: - new_exp = [] - for f in exp: - new_exp.append( + if not replace_in_path: + return file_path + + # now we need to switch scene in expected files + # because token will now point to published + # scene file and that might differ from current one + new_scene = os.path.splitext(os.path.basename(file_path))[0] + orig_scene = os.path.splitext(os.path.basename( + self._instance.context.data["currentFile"]))[0] + exp = self._instance.data.get("expectedFiles") + + if isinstance(exp[0], dict): + # we have aovs and we need to iterate over them + new_exp = {} + for aov, files in exp[0].items(): + replaced_files = [] + for f in files: + replaced_files.append( str(f).replace(orig_scene, new_scene) ) - self._instance.data["expectedFiles"] = new_exp + new_exp[aov] = replaced_files + # [] might be too much here, TODO + self._instance.data["expectedFiles"] = [new_exp] + else: + new_exp = [] + for f in exp: + new_exp.append( + str(f).replace(orig_scene, new_scene) + ) + self._instance.data["expectedFiles"] = new_exp - self.log.info("Scene name was switched {} -> {}".format( - orig_scene, new_scene - )) + self.log.info("Scene name was switched {} -> {}".format( + orig_scene, new_scene + )) return file_path From e81e3a7a1021db4e442aa3147ed03ccf8d92d8c6 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 29 Aug 2022 19:22:06 +0200 Subject: [PATCH 027/102] Fix missing line --- openpype/modules/deadline/abstract_submit_deadline.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py index 55e16d8d21..86eebc0d35 100644 --- a/openpype/modules/deadline/abstract_submit_deadline.py +++ b/openpype/modules/deadline/abstract_submit_deadline.py @@ -547,7 +547,10 @@ class AbstractSubmitDeadline(pyblish.api.InstancePlugin): file_path = None for i in self._instance.context: - is_workfile = + is_workfile = ( + "workfile" in i.data.get("families", []) or + i.data["family"] == "workfile" + ) if not is_workfile: continue From 2933b37ef7711aac4e04284120b6c9b0ce2c9612 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 29 Aug 2022 19:34:27 +0200 Subject: [PATCH 028/102] Refactor code for readability --- .../deadline/abstract_submit_deadline.py | 124 ++++++++++-------- 1 file changed, 68 insertions(+), 56 deletions(-) diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py index 86eebc0d35..46baa9ee57 100644 --- a/openpype/modules/deadline/abstract_submit_deadline.py +++ b/openpype/modules/deadline/abstract_submit_deadline.py @@ -543,72 +543,84 @@ class AbstractSubmitDeadline(pyblish.api.InstancePlugin): published. """ - anatomy = self._instance.context.data['anatomy'] - file_path = None - for i in self._instance.context: - is_workfile = ( - "workfile" in i.data.get("families", []) or - i.data["family"] == "workfile" - ) - if not is_workfile: - continue + def _get_workfile_instance(context): + """Find workfile instance in context""" + for i in context: - # test if there is instance of workfile waiting - # to be published. - assert i.data["publish"] is True, ( - "Workfile (scene) must be published along") - # determine published path from Anatomy. - template_data = i.data.get("anatomyData") - rep = i.data.get("representations")[0] - template_data["representation"] = rep.get("name") - template_data["ext"] = rep.get("ext") - template_data["comment"] = None - anatomy_filled = anatomy.format(template_data) - template_filled = anatomy_filled["publish"]["path"] - file_path = os.path.normpath(template_filled) + is_workfile = ( + "workfile" in i.data.get("families", []) or + i.data["family"] == "workfile" + ) + if not is_workfile: + continue - self.log.info("Using published scene for render {}".format( - file_path)) + # test if there is instance of workfile waiting + # to be published. + assert i.data["publish"] is True, ( + "Workfile (scene) must be published along") - if not os.path.exists(file_path): - self.log.error("published scene does not exist!") - raise + return i - if not replace_in_path: - return file_path + instance = self._instance + workfile_instance = _get_workfile_instance(instance.context) + if not workfile_instance: + return - # now we need to switch scene in expected files - # because token will now point to published - # scene file and that might differ from current one - new_scene = os.path.splitext(os.path.basename(file_path))[0] - orig_scene = os.path.splitext(os.path.basename( - self._instance.context.data["currentFile"]))[0] - exp = self._instance.data.get("expectedFiles") + # determine published path from Anatomy. + template_data = workfile_instance.data.get("anatomyData") + rep = workfile_instance.data.get("representations")[0] + template_data["representation"] = rep.get("name") + template_data["ext"] = rep.get("ext") + template_data["comment"] = None - if isinstance(exp[0], dict): - # we have aovs and we need to iterate over them - new_exp = {} - for aov, files in exp[0].items(): - replaced_files = [] - for f in files: - replaced_files.append( - str(f).replace(orig_scene, new_scene) - ) - new_exp[aov] = replaced_files - # [] might be too much here, TODO - self._instance.data["expectedFiles"] = [new_exp] - else: - new_exp = [] - for f in exp: - new_exp.append( + anatomy = instance.context.data['anatomy'] + anatomy_filled = anatomy.format(template_data) + template_filled = anatomy_filled["publish"]["path"] + file_path = os.path.normpath(template_filled) + + self.log.info("Using published scene for render {}".format(file_path)) + + if not os.path.exists(file_path): + self.log.error("published scene does not exist!") + raise + + if not replace_in_path: + return file_path + + # now we need to switch scene in expected files + # because token will now point to published + # scene file and that might differ from current one + def _clean_name(path): + return os.path.splitext(os.path.basename(path))[0] + + new_scene = _clean_name(file_path) + orig_scene = _clean_name(instance.context.data["currentFile"]) + expected_files = instance.data.get("expectedFiles") + + if isinstance(expected_files[0], dict): + # we have aovs and we need to iterate over them + new_exp = {} + for aov, files in expected_files[0].items(): + replaced_files = [] + for f in files: + replaced_files.append( str(f).replace(orig_scene, new_scene) ) - self._instance.data["expectedFiles"] = new_exp + new_exp[aov] = replaced_files + # [] might be too much here, TODO + instance.data["expectedFiles"] = [new_exp] + else: + new_exp = [] + for f in expected_files: + new_exp.append( + str(f).replace(orig_scene, new_scene) + ) + instance.data["expectedFiles"] = new_exp - self.log.info("Scene name was switched {} -> {}".format( - orig_scene, new_scene - )) + self.log.info("Scene name was switched {} -> {}".format( + orig_scene, new_scene + )) return file_path From c725ff5b42c2f3f248a6af8f835020c9efb23182 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 29 Aug 2022 19:35:05 +0200 Subject: [PATCH 029/102] Move replacing in `publishRenderMetadataFolder` to abstract base class --- openpype/modules/deadline/abstract_submit_deadline.py | 6 ++++++ .../deadline/plugins/publish/submit_maya_deadline.py | 9 --------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py index 46baa9ee57..f56cf49f6d 100644 --- a/openpype/modules/deadline/abstract_submit_deadline.py +++ b/openpype/modules/deadline/abstract_submit_deadline.py @@ -618,6 +618,12 @@ class AbstractSubmitDeadline(pyblish.api.InstancePlugin): ) instance.data["expectedFiles"] = new_exp + metadata_folder = instance.data.get("publishRenderMetadataFolder") + if metadata_folder: + metadata_folder = metadata_folder.replace(orig_scene, + new_scene) + instance.data["publishRenderMetadataFolder"] = metadata_folder + self.log.info("Scene name was switched {} -> {}".format( orig_scene, new_scene )) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 07ed237c94..26c26a124c 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -258,15 +258,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): ) self._patch_workfile(filepath, patches) - # todo: on self.use_published replace path for publishRenderMetadataFolder - # if instance.data.get("publishRenderMetadataFolder"): - # instance.data["publishRenderMetadataFolder"] = \ - # instance.data["publishRenderMetadataFolder"].replace( - # orig_scene, new_scene) - # self.log.info("Scene name was switched {} -> {}".format( - # orig_scene, new_scene - # )) - # Gather needed data ------------------------------------------------ workspace = context.data["workspaceDir"] default_render_file = instance.context.data.get('project_settings')\ From 23e652a51f41a6c65d6feba6edb0268f85feccb8 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 29 Aug 2022 19:54:36 +0200 Subject: [PATCH 030/102] Patch plug-in payload with settings --- .../maya/plugins/publish/collect_render.py | 1 + .../plugins/publish/submit_maya_deadline.py | 17 +++++++++-------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/collect_render.py b/openpype/hosts/maya/plugins/publish/collect_render.py index ebda5e190d..768a53329f 100644 --- a/openpype/hosts/maya/plugins/publish/collect_render.py +++ b/openpype/hosts/maya/plugins/publish/collect_render.py @@ -293,6 +293,7 @@ class CollectMayaRender(pyblish.api.ContextPlugin): "source": filepath, "expectedFiles": full_exp_files, "publishRenderMetadataFolder": common_publish_meta_path, + "renderProducts": layer_render_products, "resolutionWidth": lib.get_attr_in_layer( "defaultResolution.width", layer=layer_name ), diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 26c26a124c..854a66eaa5 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -215,16 +215,21 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): renderlayer = instance.data['setMembers'] # rs_beauty - self.payload_skeleton["PluginInfo"]["RenderLayer"] = renderlayer - self.payload_skeleton["PluginInfo"]["RenderSetupIncludeLights"] = instance.data.get("renderSetupIncludeLights") # noqa - # Output driver to render plugin_info = DeadlinePluginInfo( SceneFile=context.data["currentFile"], Version=cmds.about(version=True), + RenderLayer=renderlayer, + RenderSetupIncludeLights=instance.data.get("renderSetupIncludeLights") # noqa ) - return attr.asdict(plugin_info) + plugin_payload = attr.asdict(plugin_info) + + # Patching with pluginInfo from settings + for key, value in self.pluginInfo.items(): + plugin_payload[key] = value + + return plugin_payload def process_submission(self): @@ -338,10 +343,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): # Store output dir for unified publisher (filesequence) instance.data["outputDir"] = os.path.dirname(output_filename_0) - # add jobInfo and pluginInfo variables from Settings - payload["JobInfo"].update(self.jobInfo) - payload["PluginInfo"].update(self.pluginInfo) - if instance.data.get("tileRendering"): # Prepare tiles data self._tile_render(instance, payload) From 4abddd027de9d4a1814ddff1b971bb9a99c47008 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 29 Aug 2022 19:55:07 +0200 Subject: [PATCH 031/102] Use collected render products for image prefix --- .../deadline/plugins/publish/submit_maya_deadline.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 854a66eaa5..bb7ae380b6 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -307,7 +307,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): "jobname": jobname, "comment": comment, "output_filename_0": output_filename_0, - "render_variables": render_variables, "renderlayer": renderlayer, "workspace": workspace, "dirname": dirname, @@ -564,6 +563,11 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): renderer = self._instance.data["renderer"] + # Get layer prefix + render_products = self._instance.data["renderProducts"] + layer_metadata = render_products.layer_data + layer_prefix = layer_metadata.filePrefix + # This hack is here because of how Deadline handles Renderman version. # it considers everything with `renderman` set as version older than # Renderman 22, and so if we are using renderman > 21 we need to set @@ -583,7 +587,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): "SceneFile": data["filepath"], # Output directory and filename "OutputFilePath": data["dirname"].replace("\\", "/"), - "OutputFilePrefix": data["render_variables"]["filename_prefix"], # noqa: E501 + "OutputFilePrefix": layer_prefix, # Only render layers are considered renderable in this pipeline "UsingRenderLayers": True, From 6f5fcecfae7ec92bf2df80fb673bfff3e1049231 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 29 Aug 2022 19:55:24 +0200 Subject: [PATCH 032/102] Use existing variable `renderer` --- .../modules/deadline/plugins/publish/submit_maya_deadline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index bb7ae380b6..c7f91905ea 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -573,7 +573,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): # Renderman 22, and so if we are using renderman > 21 we need to set # renderer string on the job to `renderman22`. We will have to change # this when Deadline releases new version handling this. - if self._instance.data["renderer"] == "renderman": + if renderer == "renderman": try: from rfm2.config import cfg # noqa except ImportError: From ae250c4a100dcc4474512913dbc858326ced3c8a Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 29 Aug 2022 19:55:40 +0200 Subject: [PATCH 033/102] Remove unused `comment` key-value --- .../modules/deadline/plugins/publish/submit_maya_deadline.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index c7f91905ea..b2b877ab0e 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -305,7 +305,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): "filename": filename, "filepath": filepath, "jobname": jobname, - "comment": comment, "output_filename_0": output_filename_0, "renderlayer": renderlayer, "workspace": workspace, From 7af7f71edacea21a00c370e4ad1e92b2fe576b66 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 29 Aug 2022 19:58:04 +0200 Subject: [PATCH 034/102] Remove logging of plugin name --- .../modules/deadline/plugins/publish/submit_maya_deadline.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index b2b877ab0e..db796f25a9 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -335,9 +335,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): if export_job: payload["JobInfo"]["JobDependency0"] = export_job - plugin = payload["JobInfo"]["Plugin"] - self.log.info("using render plugin : {}".format(plugin)) - # Store output dir for unified publisher (filesequence) instance.data["outputDir"] = os.path.dirname(output_filename_0) From f91e33c0385762a7a73cca192f4d36716377ee1e Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 29 Aug 2022 23:21:30 +0200 Subject: [PATCH 035/102] More refactoring/cleanup (WIP) --- .../plugins/publish/submit_maya_deadline.py | 304 +++++++----------- 1 file changed, 110 insertions(+), 194 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index db796f25a9..8f12a9518f 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -70,7 +70,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): tile_assembler_plugin = "OpenPypeTileAssembler" priority = 50 tile_priority = 50 - limit_groups = [] + limit = [] # limit groups jobInfo = {} pluginInfo = {} group = "none" @@ -112,23 +112,18 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): job_info.ChunkSize = instance.data.get("chunkSize", 10) job_info.Comment = context.data.get("comment") job_info.Priority = instance.data.get("priority", self.priority) + job_info.FramesPerTask = instance.data.get("framesPerTask", 1) if self.group != "none" and self.group: job_info.Group = self.group - if self.limit_groups: - job_info.LimitGroups = ",".join(self.limit_groups) + if self.limit: + job_info.LimitGroups = ",".join(self.limit) - # Optional, enable double-click to preview rendered - # frames from Deadline Monitor - self.payload_skeleton["JobInfo"]["OutputDirectory0"] = \ - os.path.dirname(output_filename_0).replace("\\", "/") - self.payload_skeleton["JobInfo"]["OutputFilename0"] = \ - output_filename_0.replace("\\", "/") - - # Add options from RenderGlobals------------------------------------- + # Add options from RenderGlobals render_globals = instance.data.get("renderGlobals", {}) - self.payload_skeleton["JobInfo"].update(render_globals) + for key, value in render_globals: + setattr(job_info, key, value) keys = [ "FTRACK_API_KEY", @@ -140,7 +135,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): "AVALON_TASK", "AVALON_APP_NAME", "OPENPYPE_DEV", - "OPENPYPE_LOG_NO_COLORS", "OPENPYPE_VERSION" ] # Add mongo url if it's enabled @@ -150,10 +144,8 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): environment = dict({key: os.environ[key] for key in keys if key in os.environ}, **legacy_io.Session) - # TODO: Taken from old publish class - test whether still needed environment["OPENPYPE_LOG_NO_COLORS"] = "1" - environment["OPENPYPE_MAYA_VERSION"] = cmds.about(v=True) # to recognize job from PYPE for turning Event On/Off environment["OPENPYPE_RENDER_JOB"] = "1" @@ -166,7 +158,10 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): ) # to recognize job from PYPE for turning Event On/Off job_info.EnvironmentKeyValue = "OPENPYPE_RENDER_JOB=1" + job_info.EnvironmentKeyValue = "OPENPYPE_LOG_NO_COLORS=1" + # Optional, enable double-click to preview rendered + # frames from Deadline Monitor for i, filepath in enumerate(instance.data["files"]): dirname = os.path.dirname(filepath) fname = os.path.basename(filepath) @@ -213,14 +208,13 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): instance = self._instance context = instance.context - renderlayer = instance.data['setMembers'] # rs_beauty - - # Output driver to render plugin_info = DeadlinePluginInfo( - SceneFile=context.data["currentFile"], + SceneFile=self.scene_path, Version=cmds.about(version=True), - RenderLayer=renderlayer, - RenderSetupIncludeLights=instance.data.get("renderSetupIncludeLights") # noqa + RenderLayer=instance.data['setMembers'], + RenderSetupIncludeLights=instance.data.get("renderSetupIncludeLights"), # noqa + ProjectPath=context.data["workspaceDir"], + UsingRenderLayers=True, ) plugin_payload = attr.asdict(plugin_info) @@ -236,12 +230,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): instance = self._instance context = instance.context - # Generated by AbstractSubmitDeadline. The `job_info`, `plugin_info` - # and `aux_files` are the skeleton payloads that are the basis for - # all the maya submissions - job_info = self.job_info - plugin_info = self.plugin_info - aux_files = self.aux_files filepath = self.scene_path # publish if `use_publish` else workfile # TODO: Avoid the need for this logic here, needed for submit publish @@ -250,18 +238,9 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): instance.data["outputDir"] = output_dir instance.data["toBeRenderedOn"] = "deadline" - self.limit_groups = self.limit - # Patch workfile (only when use_published is enabled) if self.use_published: - patches = ( - context.data["project_settings"].get( - "deadline", {}).get( - "publish", {}).get( - "MayaSubmitDeadline", {}).get( - "scene_patches", {}) - ) - self._patch_workfile(filepath, patches) + self._patch_workfile() # Gather needed data ------------------------------------------------ workspace = context.data["workspaceDir"] @@ -271,22 +250,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): .get('default_render_image_folder') filename = os.path.basename(filepath) dirname = os.path.join(workspace, default_render_file) - renderlayer = instance.data['setMembers'] # rs_beauty - - # Get the variables depending on the renderer - # TODO: Find replacement logic for `get_renderer_variables` through - # what is collected for the render or is implemented in maya - # api `lib_renderproducts` - render_variables = get_renderer_variables(renderlayer, dirname) - filename_0 = render_variables["filename_0"] - if self.use_published: - new_scene = os.path.splitext(filename)[0] - orig_scene = os.path.splitext( - os.path.basename(context.data["currentFile"]))[0] - filename_0 = render_variables["filename_0"].replace( - orig_scene, new_scene) - - output_filename_0 = filename_0 # this is needed because renderman handles directory and file # prefixes separately @@ -301,16 +264,18 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): pass # Fill in common data to payload ------------------------------------ + # TODO: Replace these with collected data from CollectRender payload_data = { "filename": filename, "filepath": filepath, - "jobname": jobname, "output_filename_0": output_filename_0, "renderlayer": renderlayer, - "workspace": workspace, "dirname": dirname, } + # Store output dir for unified publisher (filesequence) + instance.data["outputDir"] = os.path.dirname(output_filename_0) + # Submit preceding export jobs ------------------------------------- export_job = None assert not all(x in instance.data["families"] @@ -333,17 +298,16 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): # Add export job as dependency -------------------------------------- if export_job: - payload["JobInfo"]["JobDependency0"] = export_job - - # Store output dir for unified publisher (filesequence) - instance.data["outputDir"] = os.path.dirname(output_filename_0) + job_info, _ = payload + job_info.JobDependency = export_job if instance.data.get("tileRendering"): # Prepare tiles data self._tile_render(instance, payload) else: # Submit main render job - self.submit(payload) + job_info, plugin_info = payload + self.submit(self.assemble_payload(job_info, plugin_info)) def _tile_render(self, instance, payload): @@ -546,18 +510,12 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): instance.data["jobBatchName"])) def _get_maya_payload(self, data): - payload = copy.deepcopy(self.payload_skeleton) - if not self.asset_dependencies: - job_info_ext = {} + job_info = copy.deepcopy(self.job_info) - else: - job_info_ext = { - # Asset dependency to wait for at least the scene file to sync. - "AssetDependency0": data["filepath"], - } - - renderer = self._instance.data["renderer"] + if self.asset_dependencies: + # Asset dependency to wait for at least the scene file to sync. + job_info.AssetDependency = self.scene_path # Get layer prefix render_products = self._instance.data["renderProducts"] @@ -569,6 +527,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): # Renderman 22, and so if we are using renderman > 21 we need to set # renderer string on the job to `renderman22`. We will have to change # this when Deadline releases new version handling this. + renderer = self._instance.data["renderer"] if renderer == "renderman": try: from rfm2.config import cfg # noqa @@ -580,29 +539,20 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): renderer = "renderman22" plugin_info = { - "SceneFile": data["filepath"], # Output directory and filename "OutputFilePath": data["dirname"].replace("\\", "/"), "OutputFilePrefix": layer_prefix, - - # Only render layers are considered renderable in this pipeline - "UsingRenderLayers": True, - - # Render only this layer - "RenderLayer": data["renderlayer"], - - # Determine which renderer to use from the file itself - "Renderer": renderer, - - # Resolve relative references - "ProjectPath": data["workspace"], } - payload["JobInfo"].update(job_info_ext) - payload["PluginInfo"].update(plugin_info) - return payload + + return job_info, plugin_info def _get_vray_export_payload(self, data): - payload = copy.deepcopy(self.payload_skeleton) + + job_info = copy.deepcopy(self.job_info) + + job_info.Name = self._job_info_label("Export") + + # Get V-Ray settings info to compute output path vray_settings = cmds.ls(type="VRaySettingsNode") node = vray_settings[0] template = cmds.getAttr("{}.vrscene_filename".format(node)) @@ -610,34 +560,15 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): first_file = self.format_vray_output_filename(scene, template) first_file = "{}/{}".format(data["workspace"], first_file) output = os.path.dirname(first_file) - job_info_ext = { - # Job name, as seen in Monitor - "Name": "Export {} [{}-{}]".format( - data["jobname"], - int(self._instance.data["frameStartHandle"]), - int(self._instance.data["frameEndHandle"])), - "Plugin": self._instance.data.get( - "mayaRenderPlugin", "MayaPype"), - "FramesPerTask": self._instance.data.get("framesPerTask", 1) - } - - plugin_info_ext = { - # Renderer + plugin_info = { "Renderer": "vray", - # Input - "SceneFile": data["filepath"], "SkipExistingFrames": True, - "UsingRenderLayers": True, "UseLegacyRenderLayers": True, - "RenderLayer": data["renderlayer"], - "ProjectPath": data["workspace"], "OutputFilePath": output } - payload["JobInfo"].update(job_info_ext) - payload["PluginInfo"].update(plugin_info_ext) - return payload + return job_info, plugin_info def _get_arnold_export_payload(self, data): @@ -653,76 +584,55 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): script = os.path.normpath(module_path) - payload = copy.deepcopy(self.payload_skeleton) - job_info_ext = { - # Job name, as seen in Monitor - "Name": "Export {} [{}-{}]".format( - data["jobname"], - int(self._instance.data["frameStartHandle"]), - int(self._instance.data["frameEndHandle"])), + job_info = copy.deepcopy(self.job_info) + plugin_info = copy.deepcopy(self.plugin_info) - "Plugin": "Python", - "FramesPerTask": self._instance.data.get("framesPerTask", 1), - "Frames": 1 + job_info.Name = self._job_info_label("Export") + + # Force a single frame Python job + job_info.Plugin = "Python" + job_info.Frames = 1 + + # add required env vars for the export script + envs = { + "AVALON_APP_NAME": os.environ.get("AVALON_APP_NAME"), + "OPENPYPE_ASS_EXPORT_RENDER_LAYER": data["renderlayer"], + "OPENPYPE_ASS_EXPORT_SCENE_FILE": self.scene_path, + "OPENPYPE_ASS_EXPORT_OUTPUT": payload['JobInfo']['OutputFilename0'], # noqa + "OPENPYPE_ASS_EXPORT_START": int(self._instance.data["frameStartHandle"]), # noqa + "OPENPYPE_ASS_EXPORT_END": int(self._instance.data["frameEndHandle"]), # noqa + "OPENPYPE_ASS_EXPORT_STEP": 1 } + for key, value in envs.items(): + job_info.EnvironmentKeyValue = "{key}={value}".format(key=key, + value=value) - plugin_info_ext = { + plugin_info.update({ "Version": "3.6", "ScriptFile": script, "Arguments": "", "SingleFrameOnly": "True", - } - payload["JobInfo"].update(job_info_ext) - payload["PluginInfo"].update(plugin_info_ext) + }) - envs = [ - v - for k, v in payload["JobInfo"].items() - if k.startswith("EnvironmentKeyValue") - ] - - # add app name to environment - envs.append( - "AVALON_APP_NAME={}".format(os.environ.get("AVALON_APP_NAME"))) - envs.append( - "OPENPYPE_ASS_EXPORT_RENDER_LAYER={}".format(data["renderlayer"])) - envs.append( - "OPENPYPE_ASS_EXPORT_SCENE_FILE={}".format(data["filepath"])) - envs.append( - "OPENPYPE_ASS_EXPORT_OUTPUT={}".format( - payload['JobInfo']['OutputFilename0'])) - envs.append( - "OPENPYPE_ASS_EXPORT_START={}".format( - int(self._instance.data["frameStartHandle"]))) - envs.append( - "OPENPYPE_ASS_EXPORT_END={}".format( - int(self._instance.data["frameEndHandle"]))) - envs.append( - "OPENPYPE_ASS_EXPORT_STEP={}".format(1)) - - for i, e in enumerate(envs): - payload["JobInfo"]["EnvironmentKeyValue{}".format(i)] = e - return payload + return job_info, plugin_info def _get_vray_render_payload(self, data): - payload = copy.deepcopy(self.payload_skeleton) + + # Job Info + job_info = copy.deepcopy(self.job_info) + job_info.Name = self._job_info_label("Render") + job_info.Plugin = "Vray" + job_info.OverrideTaskExtraInfoNames = False + + # Plugin Info vray_settings = cmds.ls(type="VRaySettingsNode") node = vray_settings[0] template = cmds.getAttr("{}.vrscene_filename".format(node)) # "vrayscene//_/" - scene, _ = os.path.splitext(data["filename"]) + scene, _ = os.path.splitext(self.scene_path) first_file = self.format_vray_output_filename(scene, template) first_file = "{}/{}".format(data["workspace"], first_file) - job_info_ext = { - "Name": "Render {} [{}-{}]".format( - data["jobname"], - int(self._instance.data["frameStartHandle"]), - int(self._instance.data["frameEndHandle"])), - - "Plugin": "Vray", - "OverrideTaskExtraInfoNames": False, - } plugin_info = { "InputFilename": first_file, @@ -731,35 +641,28 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): "Width": self._instance.data["resolutionWidth"], "Height": self._instance.data["resolutionHeight"], - "OutputFilePath": payload["JobInfo"]["OutputDirectory0"], - "OutputFileName": payload["JobInfo"]["OutputFilename0"] + "OutputFilePath": job_info.OutputDirectory[0], + "OutputFileName": job_info.OutputFilename[0] } - payload["JobInfo"].update(job_info_ext) - payload["PluginInfo"].update(plugin_info) - return payload + return job_info, plugin_info def _get_arnold_render_payload(self, data): - payload = copy.deepcopy(self.payload_skeleton) + + # Job Info + job_info = copy.deepcopy(self.job_info) + job_info.Name = self._job_info_label("Render") + job_info.Plugin = "Arnold" + job_info.OverrideTaskExtraInfoNames = False + + # Plugin Info ass_file, _ = os.path.splitext(data["output_filename_0"]) first_file = ass_file + ".ass" - job_info_ext = { - "Name": "Render {} [{}-{}]".format( - data["jobname"], - int(self._instance.data["frameStartHandle"]), - int(self._instance.data["frameEndHandle"])), - - "Plugin": "Arnold", - "OverrideTaskExtraInfoNames": False, - } - plugin_info = { "ArnoldFile": first_file, } - payload["JobInfo"].update(job_info_ext) - payload["PluginInfo"].update(plugin_info) - return payload + return job_info, plugin_info def format_vray_output_filename(self, filename, template, dir=False): """Format the expected output file of the Export job. @@ -804,7 +707,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): return result - def _patch_workfile(self, file, patches): + def _patch_workfile(self): # type: (str, dict) -> [str, None] """Patch Maya scene. @@ -818,19 +721,25 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): "line": "line to insert" } - Args: - file (str): File to patch. - patches (dict): Dictionary defining patches. - - Returns: - str: Patched file path or None - """ - if not patches or os.path.splitext(file)[1].lower() != ".ma": + project_settings = self._instance.context.data["project_settings"] + patches = ( + project_settings.get( + "deadline", {}).get( + "publish", {}).get( + "MayaSubmitDeadline", {}).get( + "scene_patches", {}) + ) + if not patches: + return + + if not os.path.splitext(self.scene_path)[1].lower() != ".ma": + self.log.debug("Skipping workfile patch since workfile is not " + ".ma file") return compiled_regex = [re.compile(p["regex"]) for p in patches] - with open(file, "r+") as pf: + with open(self.scene_path, "r+") as pf: scene_data = pf.readlines() for ln, line in enumerate(scene_data): for i, r in enumerate(compiled_regex): @@ -839,10 +748,17 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): pf.seek(0) pf.writelines(scene_data) pf.truncate() - self.log.info( - "Applied {} patch to scene.".format( - patches[i]["name"])) - return file + self.log.info("Applied {} patch to scene.".format( + patches[i]["name"] + )) + + def _job_info_label(self, label): + return "{label} {job.Name} [{start}-{end}]".format( + label=label, + job=self.job_info, + start=int(self._instance.data["frameStartHandle"]), + end=int(self._instance.data["frameEndHandle"]), + ) def _format_tiles( From f9bbda244bee373dd3bfb025528923d061808525 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 29 Aug 2022 23:22:19 +0200 Subject: [PATCH 036/102] More explicit PluginInfo name --- .../modules/deadline/plugins/publish/submit_maya_deadline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 8f12a9518f..87ef4e6db9 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -38,7 +38,7 @@ from openpype_modules.deadline.abstract_submit_deadline import DeadlineJobInfo @attr.s -class DeadlinePluginInfo(): +class MayaPluginInfo: SceneFile = attr.ib(default=None) # Input OutputFilePath = attr.ib(default=None) # Output directory and filename OutputFilePrefix = attr.ib(default=None) @@ -208,7 +208,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): instance = self._instance context = instance.context - plugin_info = DeadlinePluginInfo( + plugin_info = MayaPluginInfo( SceneFile=self.scene_path, Version=cmds.about(version=True), RenderLayer=instance.data['setMembers'], From ecf2a89081f19e14c65b0fd7b1992fe80519e983 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 29 Aug 2022 23:39:11 +0200 Subject: [PATCH 037/102] More temp restructuring --- .../plugins/publish/submit_maya_deadline.py | 52 +++++++------------ 1 file changed, 20 insertions(+), 32 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 87ef4e6db9..a77ccd73d4 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -273,9 +273,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): "dirname": dirname, } - # Store output dir for unified publisher (filesequence) - instance.data["outputDir"] = os.path.dirname(output_filename_0) - # Submit preceding export jobs ------------------------------------- export_job = None assert not all(x in instance.data["families"] @@ -326,26 +323,19 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): plugin_info["ImageWidth"] = instance.data.get("resolutionWidth") plugin_info["RegionRendering"] = True - assembly_payload = { - "AuxFiles": [], - "JobInfo": { - "BatchName": payload["JobInfo"]["BatchName"], - "Frames": 1, - "Name": "{} - Tile Assembly Job".format( - payload["JobInfo"]["Name"]), - "OutputDirectory0": - payload["JobInfo"]["OutputDirectory0"].replace( - "\\", "/"), - "Plugin": self.tile_assembler_plugin, - "MachineLimit": 1 - }, - "PluginInfo": { + assembly_job_info = copy.deepcopy(job_info) + assembly_job_info.Plugin = self.tile_assembler_plugin + assembly_job_info.Name = "{job.Name} - Tile Assembly Job".format( + job=job_info) + assembly_job_info.Frames = 1 + assembly_job_info.MachineLimit = 1 + assembly_job_info.Priority = instance.data.get("tile_priority", + self.tile_priority) + + assembly_plugin_info = { "CleanupTiles": 1, "ErrorOnMissing": True - } } - assembly_payload["JobInfo"]["Priority"] = self._instance.data.get( - "tile_priority", self.tile_priority) frame_payloads = [] assembly_payloads = [] @@ -414,6 +404,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): file_index = 1 for file in assembly_files: frame = re.search(R_FRAME_NUMBER, file).group("frame") + new_assembly_payload = copy.deepcopy(assembly_payload) new_assembly_payload["JobInfo"]["Name"] = \ "{} (Frame {})".format( @@ -434,7 +425,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): self.log.info( "Submitting tile job(s) [{}] ...".format(len(frame_payloads))) - url = "{}/api/jobs".format(self.deadline_url) tiles_count = instance.data.get("tilesX") * instance.data.get( "tilesY") # noqa: E501 @@ -444,9 +434,11 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): job_id = response.json()["_id"] hash = response.json()["Props"]["Ex0"] + # Add assembly job dependencies for assembly_job in assembly_payloads: - if assembly_job["JobInfo"]["ExtraInfo0"] == hash: - assembly_job["JobInfo"]["JobDependency0"] = job_id + assembly_job_info = assembly_job["JobInfo"] + if assembly_job_info.ExtraInfo[0] == hash: + assembly_job.JobDependency = job_id for assembly_job in assembly_payloads: file = assembly_job["JobInfo"]["ExtraInfo1"] @@ -461,14 +453,14 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): ) ) + config_file_dir = os.path.dirname(config_file) try: - if not os.path.isdir(os.path.dirname(config_file)): - os.makedirs(os.path.dirname(config_file)) + if not os.path.isdir(config_file_dir): + os.makedirs(config_file_dir) except OSError: # directory is not available - self.log.warning( - "Path is unreachable: `{}`".format( - os.path.dirname(config_file))) + self.log.warning("Path is unreachable: " + "`{}`".format(config_file_dir)) # add config file as job auxFile assembly_job["AuxFiles"] = [config_file] @@ -505,10 +497,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): response.json()["_id"]) job_idx += 1 - instance.data["jobBatchName"] = payload["JobInfo"]["BatchName"] - self.log.info("Setting batch name on instance: {}".format( - instance.data["jobBatchName"])) - def _get_maya_payload(self, data): job_info = copy.deepcopy(self.job_info) From 6abafd0aca1ca06204f5e5bc11907a0a6a855900 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 30 Aug 2022 00:41:30 +0200 Subject: [PATCH 038/102] Refactor tile logic --- .../plugins/publish/submit_maya_deadline.py | 120 ++++++++---------- 1 file changed, 55 insertions(+), 65 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index a77ccd73d4..920adf7e4a 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -314,11 +314,12 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): # if we have sequence of files, we need to create tile job for # every frame - job_info.TileJob = True job_info.TileJobTilesInX = instance.data.get("tilesX") job_info.TileJobTilesInY = instance.data.get("tilesY") + tiles_count = job_info.TileJobTilesInX * job_info.TileJobTilesInY + plugin_info["ImageHeight"] = instance.data.get("resolutionHeight") plugin_info["ImageWidth"] = instance.data.get("resolutionWidth") plugin_info["RegionRendering"] = True @@ -334,7 +335,8 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): assembly_plugin_info = { "CleanupTiles": 1, - "ErrorOnMissing": True + "ErrorOnMissing": True, + "Renderer": self._instance.data["renderer"] } frame_payloads = [] @@ -367,81 +369,69 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): file_index = 1 for file in files: frame = re.search(R_FRAME_NUMBER, file).group("frame") - new_payload = copy.deepcopy(payload) - new_payload["JobInfo"]["Name"] = \ - "{} (Frame {} - {} tiles)".format( + + new_job_info = copy.deepcopy(job_info) + new_job_info.Name = "{} (Frame {} - {} tiles)".format( payload["JobInfo"]["Name"], frame, instance.data.get("tilesX") * instance.data.get("tilesY") - # noqa: E501 - ) - self.log.info( - "... preparing job {}".format( - new_payload["JobInfo"]["Name"])) - new_payload["JobInfo"]["TileJobFrame"] = frame + ) + new_job_info.TileJobFrame = frame - tiles_data = _format_tiles( + new_plugin_info = copy.deepcopy(plugin_info) + + # Add tile data into job info and plugin info + tiles_out, _ = _format_tiles( file, 0, instance.data.get("tilesX"), instance.data.get("tilesY"), instance.data.get("resolutionWidth"), instance.data.get("resolutionHeight"), payload["PluginInfo"]["OutputFilePrefix"] - )[0] - new_payload["JobInfo"].update(tiles_data["JobInfo"]) - new_payload["PluginInfo"].update(tiles_data["PluginInfo"]) + ) + new_job_info.update(tiles_out["JobInfo"]) + new_plugin_info.update(tiles_out["PluginInfo"]) self.log.info("hashing {} - {}".format(file_index, file)) job_hash = hashlib.sha256( ("{}_{}".format(file_index, file)).encode("utf-8")) frame_jobs[frame] = job_hash.hexdigest() - new_payload["JobInfo"]["ExtraInfo0"] = job_hash.hexdigest() - new_payload["JobInfo"]["ExtraInfo1"] = file - frame_payloads.append(new_payload) - file_index += 1 + new_job_info.ExtraInfo[0] = job_hash.hexdigest() + new_job_info.ExtraInfo[1] = file - file_index = 1 - for file in assembly_files: - frame = re.search(R_FRAME_NUMBER, file).group("frame") - - new_assembly_payload = copy.deepcopy(assembly_payload) - new_assembly_payload["JobInfo"]["Name"] = \ - "{} (Frame {})".format( - assembly_payload["JobInfo"]["Name"], - frame) - new_assembly_payload["JobInfo"]["OutputFilename0"] = re.sub( - REPL_FRAME_NUMBER, - "\\1{}\\3".format("#" * len(frame)), file) - - new_assembly_payload["PluginInfo"]["Renderer"] = \ - self._instance.data["renderer"] # noqa: E501 - new_assembly_payload["JobInfo"]["ExtraInfo0"] = frame_jobs[ - frame] # noqa: E501 - new_assembly_payload["JobInfo"]["ExtraInfo1"] = file - assembly_payloads.append(new_assembly_payload) + frame_payloads.append(self.assemble_payload( + job_info=new_job_info, + plugin_info=new_plugin_info + )) file_index += 1 self.log.info( "Submitting tile job(s) [{}] ...".format(len(frame_payloads))) - tiles_count = instance.data.get("tilesX") * instance.data.get( - "tilesY") # noqa: E501 - - for tile_job in frame_payloads: - response = self.submit(tile_job) - + frame_tile_job_id = {} + for tile_job_payload in frame_payloads: + response = self.submit(tile_job_payload) job_id = response.json()["_id"] - hash = response.json()["Props"]["Ex0"] + frame_tile_job_id[frame] = job_id - # Add assembly job dependencies - for assembly_job in assembly_payloads: - assembly_job_info = assembly_job["JobInfo"] - if assembly_job_info.ExtraInfo[0] == hash: - assembly_job.JobDependency = job_id + assembly_jobs = [] + for i, file in enumerate(assembly_files): + frame = re.search(R_FRAME_NUMBER, file).group("frame") + + frame_assembly_job_info = copy.deepcopy(assembly_job_info) + frame_assembly_job_info.Name += " (Frame {})".format(frame) + frame_assembly_job_info.OutputFilename[0] = re.sub( + REPL_FRAME_NUMBER, + "\\1{}\\3".format("#" * len(frame)), file) + + hash = frame_jobs[frame] + tile_job_id = frame_tile_job_id[frame] + + frame_assembly_job_info.ExtraInfo[0] = hash + frame_assembly_job_info.ExtraInfo[1] = file + frame_assembly_job_info.JobDependency = tile_job_id - for assembly_job in assembly_payloads: - file = assembly_job["JobInfo"]["ExtraInfo1"] # write assembly job config files now = datetime.now() @@ -462,9 +452,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): self.log.warning("Path is unreachable: " "`{}`".format(config_file_dir)) - # add config file as job auxFile - assembly_job["AuxFiles"] = [config_file] - with open(config_file, "w") as cf: print("TileCount={}".format(tiles_count), file=cf) print("ImageFileName={}".format(file), file=cf) @@ -485,17 +472,20 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): for k, v in tiles.items(): print("{}={}".format(k, v), file=cf) - job_idx = 1 - instance.data["assemblySubmissionJobs"] = [] - for ass_job in assembly_payloads: - self.log.info("submitting assembly job {} of {}".format( - job_idx, len(assembly_payloads) - )) - response = self.submit(ass_job) + payload = self.assemble_payload( + job_info=frame_assembly_job_info, + plugin_info=assembly_plugin_info.copy(), + # add config file as job auxFile + aux_files=[config_file] + ) - instance.data["assemblySubmissionJobs"].append( - response.json()["_id"]) - job_idx += 1 + self.log.info("submitting assembly job {} of {}".format( + i+1, len(assembly_payloads) + )) + response = self.submit(payload) + assembly_jobs.append(response.json()["_id"]) + + instance.data["assemblySubmissionJobs"] = assembly_jobs def _get_maya_payload(self, data): From a6002de641e1ad500192be433f620f17680ea056 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 30 Aug 2022 00:58:23 +0200 Subject: [PATCH 039/102] Refactor _format_tiles for readability --- .../plugins/publish/submit_maya_deadline.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 920adf7e4a..00d8eb7859 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -800,23 +800,23 @@ def _format_tiles( tiles_x, tiles_y ) - out_tile_index = "OutputFilename{}Tile{}".format( - str(index), tile - ) + + # Job Info new_filename = "{}/{}{}".format( os.path.dirname(filename), tile_prefix, os.path.basename(filename) ) - out["JobInfo"][out_tile_index] = new_filename - out["PluginInfo"]["RegionPrefix{}".format(tile)] = \ - "/{}".format(tile_prefix).join(prefix.rsplit("/", 1)) + out["JobInfo"]["OutputFilename{}Tile{}".format(index, tile)] = new_filename # noqa + # Plugin Info + out["PluginInfo"]["RegionPrefix{}".format(tile)] = "/{}".format(tile_prefix).join(prefix.rsplit("/", 1)) # noqa: E501 out["PluginInfo"]["RegionTop{}".format(tile)] = int(height) - (tile_y * h_space) # noqa: E501 out["PluginInfo"]["RegionBottom{}".format(tile)] = int(height) - ((tile_y - 1) * h_space) - 1 # noqa: E501 out["PluginInfo"]["RegionLeft{}".format(tile)] = (tile_x - 1) * w_space # noqa: E501 out["PluginInfo"]["RegionRight{}".format(tile)] = (tile_x * w_space) - 1 # noqa: E501 + # Tile config cfg["Tile{}".format(tile)] = new_filename cfg["Tile{}Tile".format(tile)] = new_filename cfg["Tile{}FileName".format(tile)] = new_filename @@ -828,4 +828,5 @@ def _format_tiles( cfg["Tile{}Height".format(tile)] = h_space tile += 1 + return out, cfg From a9fe806fec1a5e5ecf98327cfa4845b8b6d3edc0 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 30 Aug 2022 01:02:24 +0200 Subject: [PATCH 040/102] Calculate once --- .../plugins/publish/submit_maya_deadline.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 00d8eb7859..d0348119dc 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -800,6 +800,10 @@ def _format_tiles( tiles_x, tiles_y ) + top = int(height) - (tile_y * h_space) + bottom = int(height) - ((tile_y - 1) * h_space) - 1 + left = (tile_x - 1) * w_space + right = (tile_x * w_space) - 1 # Job Info new_filename = "{}/{}{}".format( @@ -811,19 +815,17 @@ def _format_tiles( # Plugin Info out["PluginInfo"]["RegionPrefix{}".format(tile)] = "/{}".format(tile_prefix).join(prefix.rsplit("/", 1)) # noqa: E501 - out["PluginInfo"]["RegionTop{}".format(tile)] = int(height) - (tile_y * h_space) # noqa: E501 - out["PluginInfo"]["RegionBottom{}".format(tile)] = int(height) - ((tile_y - 1) * h_space) - 1 # noqa: E501 - out["PluginInfo"]["RegionLeft{}".format(tile)] = (tile_x - 1) * w_space # noqa: E501 - out["PluginInfo"]["RegionRight{}".format(tile)] = (tile_x * w_space) - 1 # noqa: E501 + out["PluginInfo"]["RegionTop{}".format(tile)] = top + out["PluginInfo"]["RegionBottom{}".format(tile)] = bottom + out["PluginInfo"]["RegionLeft{}".format(tile)] = left + out["PluginInfo"]["RegionRight{}".format(tile)] = right # Tile config cfg["Tile{}".format(tile)] = new_filename cfg["Tile{}Tile".format(tile)] = new_filename cfg["Tile{}FileName".format(tile)] = new_filename - cfg["Tile{}X".format(tile)] = (tile_x - 1) * w_space - - cfg["Tile{}Y".format(tile)] = int(height) - (tile_y * h_space) - + cfg["Tile{}X".format(tile)] = left + cfg["Tile{}Y".format(tile)] = top cfg["Tile{}Width".format(tile)] = w_space cfg["Tile{}Height".format(tile)] = h_space From d7c72f97b30f85aca15b4a8148c140595f0b2a3a Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 30 Aug 2022 01:10:19 +0200 Subject: [PATCH 041/102] Batch submit assembly jobs --- .../plugins/publish/submit_maya_deadline.py | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index d0348119dc..265c0f79ec 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -339,7 +339,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): "Renderer": self._instance.data["renderer"] } - frame_payloads = [] assembly_payloads = [] R_FRAME_NUMBER = re.compile( @@ -358,14 +357,15 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): itertools.chain.from_iterable( [f for _, f in exp[0].items()])) if not files: - # if beauty doesn't exists, use first aov we found + # if beauty doesn't exist, use first aov we found files = exp[0].get(list(exp[0].keys())[0]) else: files = exp assembly_files = files + # Define frame tile jobs frame_jobs = {} - + frame_payloads = {} file_index = 1 for file in files: frame = re.search(R_FRAME_NUMBER, file).group("frame") @@ -400,22 +400,24 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): new_job_info.ExtraInfo[0] = job_hash.hexdigest() new_job_info.ExtraInfo[1] = file - frame_payloads.append(self.assemble_payload( + frame_payloads[frame] = self.assemble_payload( job_info=new_job_info, plugin_info=new_plugin_info - )) + ) file_index += 1 self.log.info( "Submitting tile job(s) [{}] ...".format(len(frame_payloads))) + # Submit frame tile jobs frame_tile_job_id = {} - for tile_job_payload in frame_payloads: + for frame, tile_job_payload in frame_payloads.items(): response = self.submit(tile_job_payload) job_id = response.json()["_id"] frame_tile_job_id[frame] = job_id - assembly_jobs = [] + # Define assembly payloads + assembly_payloads = [] for i, file in enumerate(assembly_files): frame = re.search(R_FRAME_NUMBER, file).group("frame") @@ -478,14 +480,18 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): # add config file as job auxFile aux_files=[config_file] ) + assembly_payloads.append(payload) + # Submit assembly jobs + assembly_job_ids = [] + for i, payload in enumerate(assembly_payloads): self.log.info("submitting assembly job {} of {}".format( i+1, len(assembly_payloads) )) response = self.submit(payload) - assembly_jobs.append(response.json()["_id"]) + assembly_job_ids.append(response.json()["_id"]) - instance.data["assemblySubmissionJobs"] = assembly_jobs + instance.data["assemblySubmissionJobs"] = assembly_job_ids def _get_maya_payload(self, data): From 965522585b98e441907480caee57af5dad92c2d2 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 30 Aug 2022 01:11:12 +0200 Subject: [PATCH 042/102] Remove redundant docstring --- .../deadline/plugins/publish/submit_maya_deadline.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 265c0f79ec..cd9f426977 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -51,16 +51,6 @@ class MayaPluginInfo: class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): - """Submit available render layers to Deadline. - - Renders are submitted to a Deadline Web Service as - supplied via settings key "DEADLINE_REST_URL". - - Attributes: - use_published (bool): Use published scene to render instead of the - one in work area. - - """ label = "Submit Render to Deadline" hosts = ["maya"] From 8af88e115723e6abc73fff279773043e4a520326 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 30 Aug 2022 09:08:44 +0200 Subject: [PATCH 043/102] More cleanup --- .../plugins/publish/submit_maya_deadline.py | 98 +++++++------------ 1 file changed, 38 insertions(+), 60 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index cd9f426977..95140a082f 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -86,8 +86,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): job_info.Name = "%s - %s" % (src_filename, instance.name) job_info.BatchName = src_filename job_info.Plugin = instance.data.get("mayaRenderPlugin", "MayaBatch") - job_info.UserName = context.data.get( - "deadlineUser", getpass.getuser()) + job_info.UserName = context.data.get("deadlineUser", getpass.getuser()) # Deadline requires integers in frame range frames = "{start}-{end}x{step}".format( @@ -134,25 +133,18 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): environment = dict({key: os.environ[key] for key in keys if key in os.environ}, **legacy_io.Session) - # TODO: Taken from old publish class - test whether still needed - environment["OPENPYPE_LOG_NO_COLORS"] = "1" # to recognize job from PYPE for turning Event On/Off environment["OPENPYPE_RENDER_JOB"] = "1" + environment["OPENPYPE_LOG_NO_COLORS"] = "1" - for key in keys: - val = environment.get(key) - if val: - job_info.EnvironmentKeyValue = "{key}={value}".format( - key=key, - value=val - ) - # to recognize job from PYPE for turning Event On/Off - job_info.EnvironmentKeyValue = "OPENPYPE_RENDER_JOB=1" - job_info.EnvironmentKeyValue = "OPENPYPE_LOG_NO_COLORS=1" + for key, value in environment.items(): + if not value: + continue + job_info.EnvironmentKeyValue = "{key}={value}".format(key=key, + value=value) - # Optional, enable double-click to preview rendered - # frames from Deadline Monitor - for i, filepath in enumerate(instance.data["files"]): + # Enable double-click to preview rendered frames from Deadline Monitor + for filepath in instance.data["files"]: dirname = os.path.dirname(filepath) fname = os.path.basename(filepath) job_info.OutputDirectory = dirname.replace("\\", "/") @@ -241,25 +233,10 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): filename = os.path.basename(filepath) dirname = os.path.join(workspace, default_render_file) - # this is needed because renderman handles directory and file - # prefixes separately - if self._instance.data["renderer"] == "renderman": - dirname = os.path.dirname(output_filename_0) - - # Create render folder ---------------------------------------------- - try: - # Ensure render folder exists - os.makedirs(dirname) - except OSError: - pass - # Fill in common data to payload ------------------------------------ # TODO: Replace these with collected data from CollectRender payload_data = { "filename": filename, - "filepath": filepath, - "output_filename_0": output_filename_0, - "renderlayer": renderlayer, "dirname": dirname, } @@ -299,8 +276,8 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): def _tile_render(self, instance, payload): # As collected by super process() - job_info = self.job_info - plugin_info = self.pluginInfo + job_info = copy.deepcopy(self.job_info) + plugin_info = copy.deepcopy(self.plugin_info) # if we have sequence of files, we need to create tile job for # every frame @@ -314,23 +291,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): plugin_info["ImageWidth"] = instance.data.get("resolutionWidth") plugin_info["RegionRendering"] = True - assembly_job_info = copy.deepcopy(job_info) - assembly_job_info.Plugin = self.tile_assembler_plugin - assembly_job_info.Name = "{job.Name} - Tile Assembly Job".format( - job=job_info) - assembly_job_info.Frames = 1 - assembly_job_info.MachineLimit = 1 - assembly_job_info.Priority = instance.data.get("tile_priority", - self.tile_priority) - - assembly_plugin_info = { - "CleanupTiles": 1, - "ErrorOnMissing": True, - "Renderer": self._instance.data["renderer"] - } - - assembly_payloads = [] - R_FRAME_NUMBER = re.compile( r".+\.(?P[0-9]+)\..+") # noqa: N806, E501 REPL_FRAME_NUMBER = re.compile( @@ -407,7 +367,23 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): frame_tile_job_id[frame] = job_id # Define assembly payloads + assembly_job_info = copy.deepcopy(job_info) + assembly_job_info.Plugin = self.tile_assembler_plugin + assembly_job_info.Name = "{job.Name} - Tile Assembly Job".format( + job=job_info) + assembly_job_info.Frames = 1 + assembly_job_info.MachineLimit = 1 + assembly_job_info.Priority = instance.data.get("tile_priority", + self.tile_priority) + + assembly_plugin_info = { + "CleanupTiles": 1, + "ErrorOnMissing": True, + "Renderer": self._instance.data["renderer"] + } + assembly_payloads = [] + output_dir = self.job_info.OutputDirectory[0] for i, file in enumerate(assembly_files): frame = re.search(R_FRAME_NUMBER, file).group("frame") @@ -427,22 +403,19 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): # write assembly job config files now = datetime.now() - config_file = os.path.join( - os.path.dirname(output_filename_0), + config_file = os.path.join(output_dir, "{}_config_{}.txt".format( os.path.splitext(file)[0], now.strftime("%Y_%m_%d_%H_%M_%S") ) ) - - config_file_dir = os.path.dirname(config_file) try: - if not os.path.isdir(config_file_dir): - os.makedirs(config_file_dir) + if not os.path.isdir(output_dir): + os.makedirs(output_dir) except OSError: # directory is not available self.log.warning("Path is unreachable: " - "`{}`".format(config_file_dir)) + "`{}`".format(output_dir)) with open(config_file, "w") as cf: print("TileCount={}".format(tiles_count), file=cf) @@ -567,17 +540,22 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): job_info.Plugin = "Python" job_info.Frames = 1 + renderlayer = self._instance.data["setMembers"] + # add required env vars for the export script envs = { "AVALON_APP_NAME": os.environ.get("AVALON_APP_NAME"), - "OPENPYPE_ASS_EXPORT_RENDER_LAYER": data["renderlayer"], + "OPENPYPE_ASS_EXPORT_RENDER_LAYER": renderlayer, "OPENPYPE_ASS_EXPORT_SCENE_FILE": self.scene_path, - "OPENPYPE_ASS_EXPORT_OUTPUT": payload['JobInfo']['OutputFilename0'], # noqa + "OPENPYPE_ASS_EXPORT_OUTPUT": job_info.OutputFilename[0], "OPENPYPE_ASS_EXPORT_START": int(self._instance.data["frameStartHandle"]), # noqa "OPENPYPE_ASS_EXPORT_END": int(self._instance.data["frameEndHandle"]), # noqa "OPENPYPE_ASS_EXPORT_STEP": 1 } for key, value in envs.items(): + if not value: + continue + job_info.EnvironmentKeyValue = "{key}={value}".format(key=key, value=value) From e8aa926cb7d338427ce7ba558f8ffa1609fde8ef Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 30 Aug 2022 09:28:42 +0200 Subject: [PATCH 044/102] Move single use of in-line function to the class for readability --- .../deadline/abstract_submit_deadline.py | 39 ++++++++++--------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py index f56cf49f6d..a3db3feac9 100644 --- a/openpype/modules/deadline/abstract_submit_deadline.py +++ b/openpype/modules/deadline/abstract_submit_deadline.py @@ -544,26 +544,8 @@ class AbstractSubmitDeadline(pyblish.api.InstancePlugin): """ - def _get_workfile_instance(context): - """Find workfile instance in context""" - for i in context: - - is_workfile = ( - "workfile" in i.data.get("families", []) or - i.data["family"] == "workfile" - ) - if not is_workfile: - continue - - # test if there is instance of workfile waiting - # to be published. - assert i.data["publish"] is True, ( - "Workfile (scene) must be published along") - - return i - instance = self._instance - workfile_instance = _get_workfile_instance(instance.context) + workfile_instance = self._get_workfile_instance(instance.context) if not workfile_instance: return @@ -689,3 +671,22 @@ class AbstractSubmitDeadline(pyblish.api.InstancePlugin): self._instance.data["deadlineSubmissionJob"] = result return result["_id"] + + @staticmethod + def _get_workfile_instance(context): + """Find workfile instance in context""" + for i in context: + + is_workfile = ( + "workfile" in i.data.get("families", []) or + i.data["family"] == "workfile" + ) + if not is_workfile: + continue + + # test if there is instance of workfile waiting + # to be published. + assert i.data["publish"] is True, ( + "Workfile (scene) must be published along") + + return i From c6a0a199e1810e5c63c484d5871d34d14594e5be Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 30 Aug 2022 09:40:08 +0200 Subject: [PATCH 045/102] Cosmetics --- .../deadline/plugins/publish/submit_maya_deadline.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 95140a082f..873005e051 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -267,15 +267,16 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): if instance.data.get("tileRendering"): # Prepare tiles data - self._tile_render(instance, payload) + self._tile_render(payload) else: # Submit main render job job_info, plugin_info = payload self.submit(self.assemble_payload(job_info, plugin_info)) - def _tile_render(self, instance, payload): + def _tile_render(self, payload): # As collected by super process() + instance = self._instance job_info = copy.deepcopy(self.job_info) plugin_info = copy.deepcopy(self.plugin_info) @@ -321,11 +322,8 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): frame = re.search(R_FRAME_NUMBER, file).group("frame") new_job_info = copy.deepcopy(job_info) - new_job_info.Name = "{} (Frame {} - {} tiles)".format( - payload["JobInfo"]["Name"], - frame, - instance.data.get("tilesX") * instance.data.get("tilesY") - ) + new_job_info.Name += " (Frame {} - {} tiles)".format(frame, + tiles_count) new_job_info.TileJobFrame = frame new_plugin_info = copy.deepcopy(plugin_info) From e429e2ec41d23c097b301c31485a38aea634d6a3 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 30 Aug 2022 10:19:38 +0200 Subject: [PATCH 046/102] Remove json dump since `renderProducts` are not serializable --- openpype/hosts/maya/plugins/publish/collect_render.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openpype/hosts/maya/plugins/publish/collect_render.py b/openpype/hosts/maya/plugins/publish/collect_render.py index 768a53329f..14aac2f206 100644 --- a/openpype/hosts/maya/plugins/publish/collect_render.py +++ b/openpype/hosts/maya/plugins/publish/collect_render.py @@ -360,7 +360,6 @@ class CollectMayaRender(pyblish.api.ContextPlugin): instance.data["label"] = label instance.data["farm"] = True instance.data.update(data) - self.log.debug("data: {}".format(json.dumps(data, indent=4))) def parse_options(self, render_globals): """Get all overrides with a value, skip those without. From 9472cbe271af4be7c0328bf05200f44c521c58ae Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 30 Aug 2022 11:23:14 +0200 Subject: [PATCH 047/102] Fix submission --- .../collect_deadline_server_from_instance.py | 2 +- .../plugins/publish/submit_maya_deadline.py | 71 +++++++++---------- 2 files changed, 33 insertions(+), 40 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/collect_deadline_server_from_instance.py b/openpype/modules/deadline/plugins/publish/collect_deadline_server_from_instance.py index a7035cd99f..9981bead3e 100644 --- a/openpype/modules/deadline/plugins/publish/collect_deadline_server_from_instance.py +++ b/openpype/modules/deadline/plugins/publish/collect_deadline_server_from_instance.py @@ -13,7 +13,7 @@ class CollectDeadlineServerFromInstance(pyblish.api.InstancePlugin): order = pyblish.api.CollectorOrder + 0.415 label = "Deadline Webservice from the Instance" - families = ["rendering"] + families = ["rendering", "renderlayer"] def process(self, instance): instance.data["deadlineUrl"] = self._collect_deadline_url(instance) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 873005e051..2afa1883c4 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -27,7 +27,6 @@ import itertools from collections import OrderedDict import attr -import clique from maya import cmds @@ -111,7 +110,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): # Add options from RenderGlobals render_globals = instance.data.get("renderGlobals", {}) - for key, value in render_globals: + for key, value in render_globals.items(): setattr(job_info, key, value) keys = [ @@ -143,13 +142,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): job_info.EnvironmentKeyValue = "{key}={value}".format(key=key, value=value) - # Enable double-click to preview rendered frames from Deadline Monitor - for filepath in instance.data["files"]: - dirname = os.path.dirname(filepath) - fname = os.path.basename(filepath) - job_info.OutputDirectory = dirname.replace("\\", "/") - job_info.OutputFilename = fname - # Adding file dependencies. if self.asset_dependencies: dependencies = instance.context.data["fileDependencies"] @@ -160,28 +152,9 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): # Add list of expected files to job # --------------------------------- exp = instance.data.get("expectedFiles") - - def _get_output_filename(files): - col, rem = clique.assemble(files) - if not col and rem: - # we couldn't find any collections but have - # individual files. - assert len(rem) == 1, ( - "Found multiple non related files " - "to render, don't know what to do " - "with them.") - return rem[0] - else: - return col[0].format('{head}{padding}{tail}') - - if isinstance(exp[0], dict): - # we have aovs and we need to iterate over them - for _aov, files in exp[0].items(): - output_file = _get_output_filename(files) - job_info.OutputFilename = output_file - else: - output_file = _get_output_filename(exp) - job_info.OutputFilename = output_file + for filepath in self._iter_expected_files(exp): + job_info.OutputDirectory = os.path.dirname(filepath) + job_info.OutputFilename = os.path.basename(filepath) return job_info @@ -194,6 +167,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): SceneFile=self.scene_path, Version=cmds.about(version=True), RenderLayer=instance.data['setMembers'], + Renderer=instance.data["renderer"], RenderSetupIncludeLights=instance.data.get("renderSetupIncludeLights"), # noqa ProjectPath=context.data["workspaceDir"], UsingRenderLayers=True, @@ -216,7 +190,9 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): # TODO: Avoid the need for this logic here, needed for submit publish # Store output dir for unified publisher (filesequence) - output_dir = os.path.dirname(instance.data["files"][0]) + expected_files = instance.data["expectedFiles"] + first_file = next(self._iter_expected_files(expected_files)) + output_dir = os.path.dirname(first_file) instance.data["outputDir"] = output_dir instance.data["toBeRenderedOn"] = "deadline" @@ -247,17 +223,20 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): "Vray Scene and Ass Scene options are mutually exclusive") if "vrayscene" in instance.data["families"]: + self.log.debug("Submitting V-Ray scene render..") vray_export_payload = self._get_vray_export_payload(payload_data) export_job = self.submit(vray_export_payload) payload = self._get_vray_render_payload(payload_data) elif "assscene" in instance.data["families"]: + self.log.debug("Submitting Arnold .ass standalone render..") ass_export_payload = self._get_arnold_export_payload(payload_data) export_job = self.submit(ass_export_payload) payload = self._get_arnold_render_payload(payload_data) else: + self.log.debug("Submitting MayaBatch render..") payload = self._get_maya_payload(payload_data) # Add export job as dependency -------------------------------------- @@ -274,6 +253,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): self.submit(self.assemble_payload(job_info, plugin_info)) def _tile_render(self, payload): + """Submit as tile render per frame with dependent assembly jobs.""" # As collected by super process() instance = self._instance @@ -315,7 +295,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): assembly_files = files # Define frame tile jobs - frame_jobs = {} + frame_file_hash = {} frame_payloads = {} file_index = 1 for file in files: @@ -343,9 +323,11 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): self.log.info("hashing {} - {}".format(file_index, file)) job_hash = hashlib.sha256( ("{}_{}".format(file_index, file)).encode("utf-8")) - frame_jobs[frame] = job_hash.hexdigest() - new_job_info.ExtraInfo[0] = job_hash.hexdigest() + file_hash = job_hash.hexdigest() + frame_file_hash[frame] = file_hash + + new_job_info.ExtraInfo[0] = file_hash new_job_info.ExtraInfo[1] = file frame_payloads[frame] = self.assemble_payload( @@ -391,10 +373,10 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): REPL_FRAME_NUMBER, "\\1{}\\3".format("#" * len(frame)), file) - hash = frame_jobs[frame] + file_hash = frame_file_hash[frame] tile_job_id = frame_tile_job_id[frame] - frame_assembly_job_info.ExtraInfo[0] = hash + frame_assembly_job_info.ExtraInfo[0] = file_hash frame_assembly_job_info.ExtraInfo[1] = file frame_assembly_job_info.JobDependency = tile_job_id @@ -483,11 +465,12 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): if int(rman_version.split(".")[0]) > 22: renderer = "renderman22" - plugin_info = { + plugin_info = copy.deepcopy(self.plugin_info) + plugin_info.update({ # Output directory and filename "OutputFilePath": data["dirname"].replace("\\", "/"), "OutputFilePrefix": layer_prefix, - } + }) return job_info, plugin_info @@ -710,6 +693,16 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): end=int(self._instance.data["frameEndHandle"]), ) + @staticmethod + def _iter_expected_files(exp): + if isinstance(exp[0], dict): + for _aov, files in exp[0].items(): + for file in files: + yield file + else: + for file in exp: + yield file + def _format_tiles( filename, index, tiles_x, tiles_y, From 227b8405479e3a87b507e63a9b59fe473d7c9276 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 30 Aug 2022 14:12:54 +0200 Subject: [PATCH 048/102] Refactor AbstractSubmitDeadline vars to allow easier access to indices --- .../deadline/abstract_submit_deadline.py | 251 +++++++----------- .../plugins/publish/submit_maya_deadline.py | 35 ++- 2 files changed, 115 insertions(+), 171 deletions(-) diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py index a3db3feac9..427faec115 100644 --- a/openpype/modules/deadline/abstract_submit_deadline.py +++ b/openpype/modules/deadline/abstract_submit_deadline.py @@ -9,6 +9,7 @@ import os from abc import abstractmethod import platform import getpass +from functools import partial from collections import OrderedDict import six @@ -66,6 +67,58 @@ def requests_get(*args, **kwargs): return requests.get(*args, **kwargs) +class DeadlineIndexedVar(dict): + """ + + Allows to set and query values by integer indices: + Query: var[1] or var.get(1) + Set: var[1] = "my_value" + Append: var += "value" + + Note: Iterating the instance is not guarantueed to be the order of the + indices. To do so iterate with `sorted()` + + """ + def __init__(self, key): + self.__key = key + + def next_available_index(self): + # Add as first unused entry + i = 0 + while i in self.keys(): + i += 1 + return i + + def serialize(self): + key = self.__key + + # Allow custom location for index in serialized string + if "{}" not in key: + key = key + "{}" + + return { + key.format(index): value for index, value in sorted(self.items()) + } + + def update(self, data): + # Force the integer key check + for key, value in data.items(): + self.__setitem__(key, value) + + def __iadd__(self, other): + index = self.next_available_index() + self[index] = other + return self + + def __setitem__(self, key, value): + if not isinstance(key, int): + raise TypeError("Key must be an integer: {}".format(key)) + + if key < 0: + raise ValueError("Negative index can't be set: {}".format(key)) + dict.__setitem__(self, key, value) + + @attr.s class DeadlineJobInfo(object): """Mapping of all Deadline *JobInfo* attributes. @@ -218,24 +271,8 @@ class DeadlineJobInfo(object): # Environment # ---------------------------------------------- - _environmentKeyValue = attr.ib(factory=list) - - @property - def EnvironmentKeyValue(self): # noqa: N802 - """Return all environment key values formatted for Deadline. - - Returns: - dict: as `{'EnvironmentKeyValue0', 'key=value'}` - - """ - out = {} - for index, v in enumerate(self._environmentKeyValue): - out["EnvironmentKeyValue{}".format(index)] = v - return out - - @EnvironmentKeyValue.setter - def EnvironmentKeyValue(self, val): # noqa: N802 - self._environmentKeyValue.append(val) + EnvironmentKeyValue = attr.ib(factory=partial(DeadlineIndexedVar, + "EnvironmentKeyValue")) IncludeEnvironment = attr.ib(default=None) # Default: false UseJobEnvironmentOnly = attr.ib(default=None) # Default: false @@ -243,142 +280,29 @@ class DeadlineJobInfo(object): # Job Extra Info # ---------------------------------------------- - _extraInfos = attr.ib(factory=list) - _extraInfoKeyValues = attr.ib(factory=list) - - @property - def ExtraInfo(self): # noqa: N802 - """Return all ExtraInfo values formatted for Deadline. - - Returns: - dict: as `{'ExtraInfo0': 'value'}` - - """ - out = {} - for index, v in enumerate(self._extraInfos): - out["ExtraInfo{}".format(index)] = v - return out - - @ExtraInfo.setter - def ExtraInfo(self, val): # noqa: N802 - self._extraInfos.append(val) - - @property - def ExtraInfoKeyValue(self): # noqa: N802 - """Return all ExtraInfoKeyValue values formatted for Deadline. - - Returns: - dict: as {'ExtraInfoKeyValue0': 'key=value'}` - - """ - out = {} - for index, v in enumerate(self._extraInfoKeyValues): - out["ExtraInfoKeyValue{}".format(index)] = v - return out - - @ExtraInfoKeyValue.setter - def ExtraInfoKeyValue(self, val): # noqa: N802 - self._extraInfoKeyValues.append(val) + ExtraInfo = attr.ib(factory=partial(DeadlineIndexedVar, "ExtraInfo")) + ExtraInfoKeyValue = attr.ib(factory=partial(DeadlineIndexedVar, + "ExtraInfoKeyValue")) # Task Extra Info Names # ---------------------------------------------- OverrideTaskExtraInfoNames = attr.ib(default=None) # Default: false - _taskExtraInfos = attr.ib(factory=list) - - @property - def TaskExtraInfoName(self): # noqa: N802 - """Return all TaskExtraInfoName values formatted for Deadline. - - Returns: - dict: as `{'TaskExtraInfoName0': 'value'}` - - """ - out = {} - for index, v in enumerate(self._taskExtraInfos): - out["TaskExtraInfoName{}".format(index)] = v - return out - - @TaskExtraInfoName.setter - def TaskExtraInfoName(self, val): # noqa: N802 - self._taskExtraInfos.append(val) + TaskExtraInfoName = attr.ib(factory=partial(DeadlineIndexedVar, + "TaskExtraInfoName")) # Output # ---------------------------------------------- - _outputFilename = attr.ib(factory=list) - _outputFilenameTile = attr.ib(factory=list) - _outputDirectory = attr.ib(factory=list) - - @property - def OutputFilename(self): # noqa: N802 - """Return all OutputFilename values formatted for Deadline. - - Returns: - dict: as `{'OutputFilename0': 'filename'}` - - """ - out = {} - for index, v in enumerate(self._outputFilename): - out["OutputFilename{}".format(index)] = v - return out - - @OutputFilename.setter - def OutputFilename(self, val): # noqa: N802 - self._outputFilename.append(val) - - @property - def OutputFilenameTile(self): # noqa: N802 - """Return all OutputFilename#Tile values formatted for Deadline. - - Returns: - dict: as `{'OutputFilenme#Tile': 'tile'}` - - """ - out = {} - for index, v in enumerate(self._outputFilenameTile): - out["OutputFilename{}Tile".format(index)] = v - return out - - @OutputFilenameTile.setter - def OutputFilenameTile(self, val): # noqa: N802 - self._outputFilenameTile.append(val) - - @property - def OutputDirectory(self): # noqa: N802 - """Return all OutputDirectory values formatted for Deadline. - - Returns: - dict: as `{'OutputDirectory0': 'dir'}` - - """ - out = {} - for index, v in enumerate(self._outputDirectory): - out["OutputDirectory{}".format(index)] = v - return out - - @OutputDirectory.setter - def OutputDirectory(self, val): # noqa: N802 - self._outputDirectory.append(val) + OutputFilename = attr.ib(factory=partial(DeadlineIndexedVar, + "OutputFilename")) + OutputFilenameTile = attr.ib(factory=partial(DeadlineIndexedVar, + "OutputFilename{}Tile")) + OutputDirectory = attr.ib(factory=partial(DeadlineIndexedVar, + "OutputDirectory")) # Asset Dependency # ---------------------------------------------- - _assetDependency = attr.ib(factory=list) - - @property - def AssetDependency(self): # noqa: N802 - """Return all OutputDirectory values formatted for Deadline. - - Returns: - dict: as `{'OutputDirectory0': 'dir'}` - - """ - out = {} - for index, v in enumerate(self._assetDependency): - out["AssetDependency{}".format(index)] = v - return out - - @OutputDirectory.setter - def AssetDependency(self, val): # noqa: N802 - self._assetDependency.append(val) + AssetDependency = attr.ib(factory=partial(DeadlineIndexedVar, + "AssetDependency")) # Tile Job # ---------------------------------------------- @@ -402,7 +326,7 @@ class DeadlineJobInfo(object): """ def filter_data(a, v): - if a.name.startswith("_"): + if isinstance(v, DeadlineIndexedVar): return False if v is None: return False @@ -410,16 +334,37 @@ class DeadlineJobInfo(object): serialized = attr.asdict( self, dict_factory=OrderedDict, filter=filter_data) - serialized.update(self.EnvironmentKeyValue) - serialized.update(self.ExtraInfo) - serialized.update(self.ExtraInfoKeyValue) - serialized.update(self.TaskExtraInfoName) - serialized.update(self.OutputFilename) - serialized.update(self.OutputFilenameTile) - serialized.update(self.OutputDirectory) - serialized.update(self.AssetDependency) + + # Custom serialize these attributes + for attribute in [ + self.EnvironmentKeyValue, + self.ExtraInfo, + self.ExtraInfoKeyValue, + self.TaskExtraInfoName, + self.OutputFilename, + self.OutputFilenameTile, + self.OutputDirectory, + self.AssetDependency + ]: + serialized.update(attribute.serialize()) + return serialized + def update(self, data): + """Update instance with data dict""" + for key, value in data.items(): + setattr(self, key, value) + + def __setattr__(self, key, value): + # Backwards compatibility: Allow appending to index vars by setting + # it on Job Info directly like: JobInfo.OutputFilename = filename + existing = getattr(self, key, None) + if isinstance(existing, DeadlineIndexedVar): + existing += value + return + + object.__setattr__(self, key, value) + @six.add_metaclass(AbstractMetaInstancePlugin) class AbstractSubmitDeadline(pyblish.api.InstancePlugin): diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 2afa1883c4..d979c92814 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -110,8 +110,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): # Add options from RenderGlobals render_globals = instance.data.get("renderGlobals", {}) - for key, value in render_globals.items(): - setattr(job_info, key, value) + job_info.update(render_globals) keys = [ "FTRACK_API_KEY", @@ -257,8 +256,10 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): # As collected by super process() instance = self._instance - job_info = copy.deepcopy(self.job_info) - plugin_info = copy.deepcopy(self.plugin_info) + + payload_job_info, payload_plugin_info = payload + job_info = copy.deepcopy(payload_job_info) + plugin_info = copy.deepcopy(payload_plugin_info) # if we have sequence of files, we need to create tile job for # every frame @@ -309,16 +310,17 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): new_plugin_info = copy.deepcopy(plugin_info) # Add tile data into job info and plugin info - tiles_out, _ = _format_tiles( + tiles_data = _format_tiles( file, 0, instance.data.get("tilesX"), instance.data.get("tilesY"), instance.data.get("resolutionWidth"), instance.data.get("resolutionHeight"), - payload["PluginInfo"]["OutputFilePrefix"] - ) - new_job_info.update(tiles_out["JobInfo"]) - new_plugin_info.update(tiles_out["PluginInfo"]) + payload_plugin_info["OutputFilePrefix"] + )[0] + + new_job_info.update(tiles_data["JobInfo"]) + new_plugin_info.update(tiles_data["PluginInfo"]) self.log.info("hashing {} - {}".format(file_index, file)) job_hash = hashlib.sha256( @@ -342,15 +344,13 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): # Submit frame tile jobs frame_tile_job_id = {} for frame, tile_job_payload in frame_payloads.items(): - response = self.submit(tile_job_payload) - job_id = response.json()["_id"] + job_id = self.submit(tile_job_payload) frame_tile_job_id[frame] = job_id # Define assembly payloads assembly_job_info = copy.deepcopy(job_info) assembly_job_info.Plugin = self.tile_assembler_plugin - assembly_job_info.Name = "{job.Name} - Tile Assembly Job".format( - job=job_info) + assembly_job_info.Name += " - Tile Assembly Job" assembly_job_info.Frames = 1 assembly_job_info.MachineLimit = 1 assembly_job_info.Priority = instance.data.get("tile_priority", @@ -411,10 +411,9 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): instance.data.get("tilesY"), instance.data.get("resolutionWidth"), instance.data.get("resolutionHeight"), - payload["PluginInfo"]["OutputFilePrefix"] + payload_plugin_info["OutputFilePrefix"] )[1] - sorted(tiles) - for k, v in tiles.items(): + for k, v in sorted(tiles.items()): print("{}={}".format(k, v), file=cf) payload = self.assemble_payload( @@ -431,8 +430,8 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): self.log.info("submitting assembly job {} of {}".format( i+1, len(assembly_payloads) )) - response = self.submit(payload) - assembly_job_ids.append(response.json()["_id"]) + assembly_job_id = self.submit(payload) + assembly_job_ids.append(assembly_job_id) instance.data["assemblySubmissionJobs"] = assembly_job_ids From a7293f2a4f6a30297eea15297f5b25100e11e9f1 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 30 Aug 2022 14:13:14 +0200 Subject: [PATCH 049/102] Fix indentation --- openpype/modules/deadline/abstract_submit_deadline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py index 427faec115..e1bdcb10d9 100644 --- a/openpype/modules/deadline/abstract_submit_deadline.py +++ b/openpype/modules/deadline/abstract_submit_deadline.py @@ -295,7 +295,7 @@ class DeadlineJobInfo(object): OutputFilename = attr.ib(factory=partial(DeadlineIndexedVar, "OutputFilename")) OutputFilenameTile = attr.ib(factory=partial(DeadlineIndexedVar, - "OutputFilename{}Tile")) + "OutputFilename{}Tile")) OutputDirectory = attr.ib(factory=partial(DeadlineIndexedVar, "OutputDirectory")) From 91a3d8494bf8b65fe37560a02018edf59433caa6 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 30 Aug 2022 14:13:39 +0200 Subject: [PATCH 050/102] Disable aux files for now since it's not supported by Deadline Webservice --- .../modules/deadline/plugins/publish/submit_maya_deadline.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index d979c92814..7694e80e9a 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -419,8 +419,9 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): payload = self.assemble_payload( job_info=frame_assembly_job_info, plugin_info=assembly_plugin_info.copy(), + # todo: aux file transfers don't work with deadline webservice # add config file as job auxFile - aux_files=[config_file] + # aux_files=[config_file] ) assembly_payloads.append(payload) From 39d216797dacad00a3f42102e146c2806b9f8244 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 30 Aug 2022 14:17:40 +0200 Subject: [PATCH 051/102] Force integer pixel values --- .../deadline/plugins/publish/submit_maya_deadline.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 7694e80e9a..3fbff0153b 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -765,10 +765,10 @@ def _format_tiles( tiles_x, tiles_y ) - top = int(height) - (tile_y * h_space) - bottom = int(height) - ((tile_y - 1) * h_space) - 1 - left = (tile_x - 1) * w_space - right = (tile_x * w_space) - 1 + top = int(height - (tile_y * h_space)) + bottom = int(height - ((tile_y - 1) * h_space) - 1) + left = int((tile_x - 1) * w_space) + right = int((tile_x * w_space) - 1) # Job Info new_filename = "{}/{}{}".format( From 2fb7cabca49bae51162e2a82dfdc7d225094ed36 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 30 Aug 2022 14:39:28 +0200 Subject: [PATCH 052/102] Shush hound --- .../deadline/abstract_submit_deadline.py | 4 ++-- .../plugins/publish/submit_maya_deadline.py | 21 +++++++++++-------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py index e1bdcb10d9..35b114da95 100644 --- a/openpype/modules/deadline/abstract_submit_deadline.py +++ b/openpype/modules/deadline/abstract_submit_deadline.py @@ -623,8 +623,8 @@ class AbstractSubmitDeadline(pyblish.api.InstancePlugin): for i in context: is_workfile = ( - "workfile" in i.data.get("families", []) or - i.data["family"] == "workfile" + "workfile" in i.data.get("families", []) or + i.data["family"] == "workfile" ) if not is_workfile: continue diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 3fbff0153b..1b69f8b4e9 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -357,14 +357,14 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): self.tile_priority) assembly_plugin_info = { - "CleanupTiles": 1, - "ErrorOnMissing": True, - "Renderer": self._instance.data["renderer"] + "CleanupTiles": 1, + "ErrorOnMissing": True, + "Renderer": self._instance.data["renderer"] } assembly_payloads = [] output_dir = self.job_info.OutputDirectory[0] - for i, file in enumerate(assembly_files): + for file in assembly_files: frame = re.search(R_FRAME_NUMBER, file).group("frame") frame_assembly_job_info = copy.deepcopy(assembly_job_info) @@ -383,7 +383,8 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): # write assembly job config files now = datetime.now() - config_file = os.path.join(output_dir, + config_file = os.path.join( + output_dir, "{}_config_{}.txt".format( os.path.splitext(file)[0], now.strftime("%Y_%m_%d_%H_%M_%S") @@ -427,10 +428,12 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): # Submit assembly jobs assembly_job_ids = [] + num_assemblies = len(assembly_payloads) for i, payload in enumerate(assembly_payloads): - self.log.info("submitting assembly job {} of {}".format( - i+1, len(assembly_payloads) - )) + self.log.info( + "submitting assembly job {} of {}".format(i + 1, + num_assemblies) + ) assembly_job_id = self.submit(payload) assembly_job_ids.append(assembly_job_id) @@ -682,7 +685,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): pf.writelines(scene_data) pf.truncate() self.log.info("Applied {} patch to scene.".format( - patches[i]["name"] + patches[i]["name"] )) def _job_info_label(self, label): From e9e01e3163079bc9f6c48fe633aed92592928328 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 30 Aug 2022 14:43:18 +0200 Subject: [PATCH 053/102] Use update method --- .../modules/deadline/plugins/publish/submit_maya_deadline.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 1b69f8b4e9..9692b136e9 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -69,8 +69,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): # todo: test whether this works for existing production cases # where custom jobInfo was stored in the project settings - for key, value in self.jobInfo.items(): - setattr(job_info, key, value) + job_info.update(self.jobInfo) instance = self._instance context = instance.context From acf3d67f242625f8a91317b602ffa6467ff7cc6b Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 31 Aug 2022 15:59:20 +0200 Subject: [PATCH 054/102] moved 'create_workdir_extra_folders' to 'openpype.pipeline.workfile' --- .../hooks/pre_create_extra_workdir_folders.py | 6 +- openpype/lib/path_tools.py | 89 ++++++++++++------- openpype/lib/plugin_tools.py | 1 - openpype/pipeline/workfile/__init__.py | 4 + openpype/pipeline/workfile/path_resolving.py | 57 ++++++++++++ openpype/tools/workfiles/files_widget.py | 10 +-- 6 files changed, 126 insertions(+), 41 deletions(-) diff --git a/openpype/hooks/pre_create_extra_workdir_folders.py b/openpype/hooks/pre_create_extra_workdir_folders.py index d79c5831ee..c5af620c87 100644 --- a/openpype/hooks/pre_create_extra_workdir_folders.py +++ b/openpype/hooks/pre_create_extra_workdir_folders.py @@ -1,8 +1,6 @@ import os -from openpype.lib import ( - PreLaunchHook, - create_workdir_extra_folders -) +from openpype.lib import PreLaunchHook +from openpype.pipeline.workfile import create_workdir_extra_folders class AddLastWorkfileToLaunchArgs(PreLaunchHook): diff --git a/openpype/lib/path_tools.py b/openpype/lib/path_tools.py index f60b2fa722..671591bca4 100644 --- a/openpype/lib/path_tools.py +++ b/openpype/lib/path_tools.py @@ -16,6 +16,51 @@ from .profiles_filtering import filter_profiles log = logging.getLogger(__name__) +class PathToolsDeprecatedWarning(DeprecationWarning): + pass + + +def deprecated(new_destination): + """Mark functions as deprecated. + + It will result in a warning being emitted when the function is used. + """ + + func = None + if callable(new_destination): + func = new_destination + new_destination = None + + def _decorator(decorated_func): + if new_destination is None: + warning_message = ( + " Please check content of deprecated function to figure out" + " possible replacement." + ) + else: + warning_message = " Please replace your usage with '{}'.".format( + new_destination + ) + + @functools.wraps(decorated_func) + def wrapper(*args, **kwargs): + warnings.simplefilter("always", PathToolsDeprecatedWarning) + warnings.warn( + ( + "Call to deprecated function '{}'" + "\nFunction was moved or removed.{}" + ).format(decorated_func.__name__, warning_message), + category=PathToolsDeprecatedWarning, + stacklevel=4 + ) + return decorated_func(*args, **kwargs) + return wrapper + + if func is None: + return _decorator + return _decorator(func) + + def format_file_size(file_size, suffix=None): """Returns formatted string with size in appropriate unit. @@ -333,6 +378,7 @@ def get_project_basic_paths(project_name): return _list_path_items(folder_structure) +@deprecated("openpype.pipeline.workfile.create_workdir_extra_folders") def create_workdir_extra_folders( workdir, host_name, task_type, task_name, project_name, project_settings=None @@ -349,37 +395,18 @@ def create_workdir_extra_folders( project_name (str): Name of project on which task is. project_settings (dict): Prepared project settings. Are loaded if not passed. + + Deprecated: + Function will be removed after release version 3.16.* """ - # Load project settings if not set - if not project_settings: - project_settings = get_project_settings(project_name) - # Load extra folders profiles - extra_folders_profiles = ( - project_settings["global"]["tools"]["Workfiles"]["extra_folders"] + from openpype.pipeline.project_folders import create_workdir_extra_folders + + return create_workdir_extra_folders( + workdir, + host_name, + task_type, + task_name, + project_name, + project_settings ) - # Skip if are empty - if not extra_folders_profiles: - return - - # Prepare profiles filters - filter_data = { - "task_types": task_type, - "task_names": task_name, - "hosts": host_name - } - profile = filter_profiles(extra_folders_profiles, filter_data) - if profile is None: - return - - for subfolder in profile["folders"]: - # Make sure backslashes are converted to forwards slashes - # and does not start with slash - subfolder = subfolder.replace("\\", "/").lstrip("/") - # Skip empty strings - if not subfolder: - continue - - fullpath = os.path.join(workdir, subfolder) - if not os.path.exists(fullpath): - os.makedirs(fullpath) diff --git a/openpype/lib/plugin_tools.py b/openpype/lib/plugin_tools.py index 81d268ea1c..1e157dfbfd 100644 --- a/openpype/lib/plugin_tools.py +++ b/openpype/lib/plugin_tools.py @@ -3,7 +3,6 @@ import os import logging import re -import json import warnings import functools diff --git a/openpype/pipeline/workfile/__init__.py b/openpype/pipeline/workfile/__init__.py index 0aad29b6f9..94ecc81bd6 100644 --- a/openpype/pipeline/workfile/__init__.py +++ b/openpype/pipeline/workfile/__init__.py @@ -9,6 +9,8 @@ from .path_resolving import ( get_custom_workfile_template, get_custom_workfile_template_by_string_context, + + create_workdir_extra_folders, ) from .build_workfile import BuildWorkfile @@ -26,5 +28,7 @@ __all__ = ( "get_custom_workfile_template", "get_custom_workfile_template_by_string_context", + "create_workdir_extra_folders", + "BuildWorkfile", ) diff --git a/openpype/pipeline/workfile/path_resolving.py b/openpype/pipeline/workfile/path_resolving.py index 6d9e72dbd2..1243e84148 100644 --- a/openpype/pipeline/workfile/path_resolving.py +++ b/openpype/pipeline/workfile/path_resolving.py @@ -467,3 +467,60 @@ def get_custom_workfile_template_by_string_context( return get_custom_workfile_template( project_doc, asset_doc, task_name, host_name, anatomy, project_settings ) + + +def create_workdir_extra_folders( + workdir, + host_name, + task_type, + task_name, + project_name, + project_settings=None +): + """Create extra folders in work directory based on context. + + Args: + workdir (str): Path to workdir where workfiles is stored. + host_name (str): Name of host implementation. + task_type (str): Type of task for which extra folders should be + created. + task_name (str): Name of task for which extra folders should be + created. + project_name (str): Name of project on which task is. + project_settings (dict): Prepared project settings. Are loaded if not + passed. + """ + + # Load project settings if not set + if not project_settings: + project_settings = get_project_settings(project_name) + + # Load extra folders profiles + extra_folders_profiles = ( + project_settings["global"]["tools"]["Workfiles"]["extra_folders"] + ) + # Skip if are empty + if not extra_folders_profiles: + return + + # Prepare profiles filters + filter_data = { + "task_types": task_type, + "task_names": task_name, + "hosts": host_name + } + profile = filter_profiles(extra_folders_profiles, filter_data) + if profile is None: + return + + for subfolder in profile["folders"]: + # Make sure backslashes are converted to forwards slashes + # and does not start with slash + subfolder = subfolder.replace("\\", "/").lstrip("/") + # Skip empty strings + if not subfolder: + continue + + fullpath = os.path.join(workdir, subfolder) + if not os.path.exists(fullpath): + os.makedirs(fullpath) diff --git a/openpype/tools/workfiles/files_widget.py b/openpype/tools/workfiles/files_widget.py index a5d5b14bb6..b4f5e422bc 100644 --- a/openpype/tools/workfiles/files_widget.py +++ b/openpype/tools/workfiles/files_widget.py @@ -10,10 +10,7 @@ from openpype.host import IWorkfileHost from openpype.client import get_asset_by_id from openpype.tools.utils import PlaceholderLineEdit from openpype.tools.utils.delegates import PrettyTimeDelegate -from openpype.lib import ( - emit_event, - create_workdir_extra_folders, -) +from openpype.lib import emit_event from openpype.pipeline import ( registered_host, legacy_io, @@ -23,7 +20,10 @@ from openpype.pipeline.context_tools import ( compute_session_changes, change_current_context ) -from openpype.pipeline.workfile import get_workfile_template_key +from openpype.pipeline.workfile import ( + get_workfile_template_key, + create_workdir_extra_folders, +) from .model import ( WorkAreaFilesModel, From d3a9610c5a92f17bd319843eab4a99f6d68acce4 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 31 Aug 2022 16:09:15 +0200 Subject: [PATCH 055/102] moved helper function to pipeline.project_folders --- openpype/lib/path_tools.py | 64 +++++++++++----------------- openpype/pipeline/project_folders.py | 49 +++++++++++++++++++++ 2 files changed, 73 insertions(+), 40 deletions(-) create mode 100644 openpype/pipeline/project_folders.py diff --git a/openpype/lib/path_tools.py b/openpype/lib/path_tools.py index 671591bca4..736eb0effc 100644 --- a/openpype/lib/path_tools.py +++ b/openpype/lib/path_tools.py @@ -277,58 +277,42 @@ def get_last_version_from_path(path_dir, filter): return None +@deprecated("openpype.pipeline.project_folders.concatenate_splitted_paths") def concatenate_splitted_paths(split_paths, anatomy): - pattern_array = re.compile(r"\[.*\]") - output = [] - for path_items in split_paths: - clean_items = [] - if isinstance(path_items, str): - path_items = [path_items] + """ + Deprecated: + Function will be removed after release version 3.16.* + """ - for path_item in path_items: - if not re.match(r"{.+}", path_item): - path_item = re.sub(pattern_array, "", path_item) - clean_items.append(path_item) + from openpype.pipeline.project_folders import concatenate_splitted_paths - # backward compatibility - if "__project_root__" in path_items: - for root, root_path in anatomy.roots.items(): - if not os.path.exists(str(root_path)): - log.debug("Root {} path path {} not exist on \ - computer!".format(root, root_path)) - continue - clean_items = ["{{root[{}]}}".format(root), - r"{project[name]}"] + clean_items[1:] - output.append(os.path.normpath(os.path.sep.join(clean_items))) - continue - - output.append(os.path.normpath(os.path.sep.join(clean_items))) - - return output + return concatenate_splitted_paths(split_paths, anatomy) +@deprecated def get_format_data(anatomy): - project_doc = get_project(anatomy.project_name, fields=["data.code"]) - project_code = project_doc["data"]["code"] + """ + Deprecated: + Function will be removed after release version 3.16.* + """ - return { - "root": anatomy.roots, - "project": { - "name": anatomy.project_name, - "code": project_code - }, - } + from openpype.pipeline.template_data import get_project_template_data + + data = get_project_template_data(project_name=anatomy.project_name) + data["root"] = anatomy.roots + return data +@deprecated("openpype.pipeline.project_folders.fill_paths") def fill_paths(path_list, anatomy): - format_data = get_format_data(anatomy) - filled_paths = [] + """ + Deprecated: + Function will be removed after release version 3.16.* + """ - for path in path_list: - new_path = path.format(**format_data) - filled_paths.append(new_path) + from openpype.pipeline.project_folders import fill_paths - return filled_paths + return fill_paths(path_list, anatomy) def create_project_folders(basic_paths, project_name): diff --git a/openpype/pipeline/project_folders.py b/openpype/pipeline/project_folders.py new file mode 100644 index 0000000000..256c4e73d8 --- /dev/null +++ b/openpype/pipeline/project_folders.py @@ -0,0 +1,49 @@ +import os +import re + +from openpype.lib import Logger + +from .template_data import get_project_template_data + + +def concatenate_splitted_paths(split_paths, anatomy): + log = Logger.get_logger("concatenate_splitted_paths") + pattern_array = re.compile(r"\[.*\]") + output = [] + for path_items in split_paths: + clean_items = [] + if isinstance(path_items, str): + path_items = [path_items] + + for path_item in path_items: + if not re.match(r"{.+}", path_item): + path_item = re.sub(pattern_array, "", path_item) + clean_items.append(path_item) + + # backward compatibility + if "__project_root__" in path_items: + for root, root_path in anatomy.roots.items(): + if not os.path.exists(str(root_path)): + log.debug("Root {} path path {} not exist on \ + computer!".format(root, root_path)) + continue + clean_items = ["{{root[{}]}}".format(root), + r"{project[name]}"] + clean_items[1:] + output.append(os.path.normpath(os.path.sep.join(clean_items))) + continue + + output.append(os.path.normpath(os.path.sep.join(clean_items))) + + return output + + +def fill_paths(path_list, anatomy): + format_data = get_project_template_data(project_name=anatomy.project_name) + format_data["root"] = anatomy.roots + filled_paths = [] + + for path in path_list: + new_path = path.format(**format_data) + filled_paths.append(new_path) + + return filled_paths From c7ffda124e36813d00757d5096475bcb902c60f3 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 31 Aug 2022 16:10:54 +0200 Subject: [PATCH 056/102] moved 'create_project_folders' and 'get_project_basic_paths' --- openpype/lib/path_tools.py | 65 ++++++------------- .../action_create_project_structure.py | 7 +- openpype/pipeline/project_folders.py | 56 ++++++++++++++++ .../project_manager/project_manager/window.py | 24 +++---- 4 files changed, 88 insertions(+), 64 deletions(-) diff --git a/openpype/lib/path_tools.py b/openpype/lib/path_tools.py index 736eb0effc..5800498b07 100644 --- a/openpype/lib/path_tools.py +++ b/openpype/lib/path_tools.py @@ -1,21 +1,17 @@ import os import re -import abc -import json import logging -import six import platform +import functools +import warnings import clique -from openpype.client import get_project -from openpype.settings import get_project_settings - -from .profiles_filtering import filter_profiles - log = logging.getLogger(__name__) + + class PathToolsDeprecatedWarning(DeprecationWarning): pass @@ -315,51 +311,28 @@ def fill_paths(path_list, anatomy): return fill_paths(path_list, anatomy) +@deprecated("openpype.pipeline.project_folders.create_project_folders") def create_project_folders(basic_paths, project_name): - from openpype.pipeline import Anatomy - anatomy = Anatomy(project_name) + """ + Deprecated: + Function will be removed after release version 3.16.* + """ - concat_paths = concatenate_splitted_paths(basic_paths, anatomy) - filled_paths = fill_paths(concat_paths, anatomy) + from openpype.pipeline.project_folders import create_project_folders - # Create folders - for path in filled_paths: - if os.path.exists(path): - log.debug("Folder already exists: {}".format(path)) - else: - log.debug("Creating folder: {}".format(path)) - os.makedirs(path) - - -def _list_path_items(folder_structure): - output = [] - for key, value in folder_structure.items(): - if not value: - output.append(key) - else: - paths = _list_path_items(value) - for path in paths: - if not isinstance(path, (list, tuple)): - path = [path] - - item = [key] - item.extend(path) - output.append(item) - - return output + return create_project_folders(project_name, basic_paths) +@deprecated("openpype.pipeline.project_folders.get_project_basic_paths") def get_project_basic_paths(project_name): - project_settings = get_project_settings(project_name) - folder_structure = ( - project_settings["global"]["project_folder_structure"] - ) - if not folder_structure: - return [] + """ + Deprecated: + Function will be removed after release version 3.16.* + """ - if isinstance(folder_structure, str): - folder_structure = json.loads(folder_structure) - return _list_path_items(folder_structure) + from openpype.pipeline.project_folders import get_project_basic_paths + + return get_project_basic_paths(project_name) @deprecated("openpype.pipeline.workfile.create_workdir_extra_folders") diff --git a/openpype/modules/ftrack/event_handlers_user/action_create_project_structure.py b/openpype/modules/ftrack/event_handlers_user/action_create_project_structure.py index df914de854..7c896570b1 100644 --- a/openpype/modules/ftrack/event_handlers_user/action_create_project_structure.py +++ b/openpype/modules/ftrack/event_handlers_user/action_create_project_structure.py @@ -1,7 +1,10 @@ import re +from openpype.pipeline.project_folders import ( + get_project_basic_paths, + create_project_folders, +) from openpype_modules.ftrack.lib import BaseAction, statics_icon -from openpype.api import get_project_basic_paths, create_project_folders class CreateProjectFolders(BaseAction): @@ -81,7 +84,7 @@ class CreateProjectFolders(BaseAction): } # Invoking OpenPype API to create the project folders - create_project_folders(basic_paths, project_name) + create_project_folders(project_name, basic_paths) self.create_ftrack_entities(basic_paths, project_entity) self.trigger_event( diff --git a/openpype/pipeline/project_folders.py b/openpype/pipeline/project_folders.py index 256c4e73d8..811b9aa648 100644 --- a/openpype/pipeline/project_folders.py +++ b/openpype/pipeline/project_folders.py @@ -1,8 +1,13 @@ import os import re +import json +import six + +from openpype.settings import get_project_settings from openpype.lib import Logger +from .anatomy import Anatomy from .template_data import get_project_template_data @@ -47,3 +52,54 @@ def fill_paths(path_list, anatomy): filled_paths.append(new_path) return filled_paths + + +def create_project_folders(project_name, basic_paths=None): + log = Logger.get_logger("create_project_folders") + anatomy = Anatomy(project_name) + if basic_paths is None: + basic_paths = get_project_basic_paths(project_name) + + concat_paths = concatenate_splitted_paths(basic_paths, anatomy) + filled_paths = fill_paths(concat_paths, anatomy) + + # Create folders + for path in filled_paths: + if os.path.exists(path): + log.debug("Folder already exists: {}".format(path)) + else: + log.debug("Creating folder: {}".format(path)) + os.makedirs(path) + return filled_paths + + +def _list_path_items(folder_structure): + output = [] + for key, value in folder_structure.items(): + if not value: + output.append(key) + continue + + paths = _list_path_items(value) + for path in paths: + if not isinstance(path, (list, tuple)): + path = [path] + + item = [key] + item.extend(path) + output.append(item) + + return output + + +def get_project_basic_paths(project_name): + project_settings = get_project_settings(project_name) + folder_structure = ( + project_settings["global"]["project_folder_structure"] + ) + if not folder_structure: + return [] + + if isinstance(folder_structure, six.string_types): + folder_structure = json.loads(folder_structure) + return _list_path_items(folder_structure) diff --git a/openpype/tools/project_manager/project_manager/window.py b/openpype/tools/project_manager/project_manager/window.py index c6ae0ff352..3b2dea8ca3 100644 --- a/openpype/tools/project_manager/project_manager/window.py +++ b/openpype/tools/project_manager/project_manager/window.py @@ -1,5 +1,12 @@ from Qt import QtWidgets, QtCore, QtGui +from openpype import resources +from openpype.style import load_stylesheet +from openpype.widgets import PasswordDialog +from openpype.lib import is_admin_password_required, Logger +from openpype.pipeline import AvalonMongoDB +from openpype.pipeline.project_folders import create_project_folders + from . import ( ProjectModel, ProjectProxyFilter, @@ -13,17 +20,6 @@ from . import ( ) from .widgets import ConfirmProjectDeletion from .style import ResourceCache -from openpype.style import load_stylesheet -from openpype.lib import is_admin_password_required -from openpype.widgets import PasswordDialog -from openpype.pipeline import AvalonMongoDB - -from openpype import resources -from openpype.api import ( - get_project_basic_paths, - create_project_folders, - Logger -) class ProjectManagerWindow(QtWidgets.QWidget): @@ -259,12 +255,8 @@ class ProjectManagerWindow(QtWidgets.QWidget): qm.Yes | qm.No) if ans == qm.Yes: try: - # Get paths based on presets - basic_paths = get_project_basic_paths(project_name) - if not basic_paths: - pass # Invoking OpenPype API to create the project folders - create_project_folders(basic_paths, project_name) + create_project_folders(project_name) except Exception as exc: self.log.warning( "Cannot create starting folders: {}".format(exc), From 27e74ee0095d33c22ee2059f12efb508662d066f Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 31 Aug 2022 16:15:28 +0200 Subject: [PATCH 057/102] skip folders creation if are not set --- openpype/pipeline/project_folders.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/openpype/pipeline/project_folders.py b/openpype/pipeline/project_folders.py index 811b9aa648..1bcba5c320 100644 --- a/openpype/pipeline/project_folders.py +++ b/openpype/pipeline/project_folders.py @@ -60,6 +60,9 @@ def create_project_folders(project_name, basic_paths=None): if basic_paths is None: basic_paths = get_project_basic_paths(project_name) + if not basic_paths: + return + concat_paths = concatenate_splitted_paths(basic_paths, anatomy) filled_paths = fill_paths(concat_paths, anatomy) @@ -70,7 +73,6 @@ def create_project_folders(project_name, basic_paths=None): else: log.debug("Creating folder: {}".format(path)) os.makedirs(path) - return filled_paths def _list_path_items(folder_structure): From 111a0be1a858f7568a169196c0be32e5d9da4a88 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 31 Aug 2022 16:21:23 +0200 Subject: [PATCH 058/102] removed not needed lines --- openpype/lib/path_tools.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/openpype/lib/path_tools.py b/openpype/lib/path_tools.py index 5800498b07..0b6d0a3391 100644 --- a/openpype/lib/path_tools.py +++ b/openpype/lib/path_tools.py @@ -10,8 +10,6 @@ import clique log = logging.getLogger(__name__) - - class PathToolsDeprecatedWarning(DeprecationWarning): pass From 1f06830a03fa58ccfa2ccf76f940ec0281343c9c Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 1 Sep 2022 13:24:20 +0200 Subject: [PATCH 059/102] change both 'user' and 'username' keys --- openpype/plugins/publish/extract_burnin.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openpype/plugins/publish/extract_burnin.py b/openpype/plugins/publish/extract_burnin.py index 88093fb92f..36a28beb5d 100644 --- a/openpype/plugins/publish/extract_burnin.py +++ b/openpype/plugins/publish/extract_burnin.py @@ -492,6 +492,7 @@ class ExtractBurnin(openpype.api.Extractor): # OPENPYPE_USERNAME might have side effects webpublish_user_name = os.environ.get("WEBPUBLISH_OPENPYPE_USERNAME") if webpublish_user_name: + burnin_data["user"] = webpublish_user_name burnin_data["username"] = webpublish_user_name self.log.debug( From 398325684961bf9b408da0e56011cfbd4eb8bed1 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 1 Sep 2022 13:24:33 +0200 Subject: [PATCH 060/102] add 'user' to representation context data --- openpype/plugins/publish/integrate.py | 2 +- openpype/plugins/publish/integrate_hero_version.py | 2 +- openpype/plugins/publish/integrate_legacy.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/openpype/plugins/publish/integrate.py b/openpype/plugins/publish/integrate.py index f99c718f8a..788966878f 100644 --- a/openpype/plugins/publish/integrate.py +++ b/openpype/plugins/publish/integrate.py @@ -135,7 +135,7 @@ class IntegrateAsset(pyblish.api.InstancePlugin): # the database even if not used by the destination template db_representation_context_keys = [ "project", "asset", "task", "subset", "version", "representation", - "family", "hierarchy", "username", "output" + "family", "hierarchy", "username", "user", "output" ] skip_host_families = [] diff --git a/openpype/plugins/publish/integrate_hero_version.py b/openpype/plugins/publish/integrate_hero_version.py index 7d698ff98d..5b8b141e88 100644 --- a/openpype/plugins/publish/integrate_hero_version.py +++ b/openpype/plugins/publish/integrate_hero_version.py @@ -46,7 +46,7 @@ class IntegrateHeroVersion(pyblish.api.InstancePlugin): ignored_representation_names = [] db_representation_context_keys = [ "project", "asset", "task", "subset", "representation", - "family", "hierarchy", "task", "username" + "family", "hierarchy", "task", "username", "user" ] # QUESTION/TODO this process should happen on server if crashed due to # permissions error on files (files were used or user didn't have perms) diff --git a/openpype/plugins/publish/integrate_legacy.py b/openpype/plugins/publish/integrate_legacy.py index b90b61f587..5a6190f38e 100644 --- a/openpype/plugins/publish/integrate_legacy.py +++ b/openpype/plugins/publish/integrate_legacy.py @@ -127,7 +127,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): exclude_families = ["render.farm"] db_representation_context_keys = [ "project", "asset", "task", "subset", "version", "representation", - "family", "hierarchy", "task", "username" + "family", "hierarchy", "task", "username", "user" ] default_template_name = "publish" From 198ad3ac901dc962ab840a0b09a5bd25d4a286e2 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 1 Sep 2022 13:25:02 +0200 Subject: [PATCH 061/102] add both 'user' and 'username' in integrate slack --- .../modules/slack/plugins/publish/integrate_slack_api.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/openpype/modules/slack/plugins/publish/integrate_slack_api.py b/openpype/modules/slack/plugins/publish/integrate_slack_api.py index c3b288f0cd..4a8e9f773f 100644 --- a/openpype/modules/slack/plugins/publish/integrate_slack_api.py +++ b/openpype/modules/slack/plugins/publish/integrate_slack_api.py @@ -95,13 +95,15 @@ class IntegrateSlackAPI(pyblish.api.InstancePlugin): Reviews might be large, so allow only adding link to message instead of uploading only. """ + fill_data = copy.deepcopy(instance.context.data["anatomyData"]) + username = fill_data.get("user") fill_pairs = [ ("asset", instance.data.get("asset", fill_data.get("asset"))), ("subset", instance.data.get("subset", fill_data.get("subset"))), - ("username", instance.data.get("username", - fill_data.get("username"))), + ("user", username), + ("username", username), ("app", instance.data.get("app", fill_data.get("app"))), ("family", instance.data.get("family", fill_data.get("family"))), ("version", str(instance.data.get("version", From 6bcbf34d27e3806ae7f62a09d7d5a6002671a2eb Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 1 Sep 2022 13:53:39 +0200 Subject: [PATCH 062/102] changed class name to 'CollectUsernameForWebpublish' --- openpype/modules/ftrack/plugins/publish/collect_username.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/openpype/modules/ftrack/plugins/publish/collect_username.py b/openpype/modules/ftrack/plugins/publish/collect_username.py index a9b746ea51..0e232bf83e 100644 --- a/openpype/modules/ftrack/plugins/publish/collect_username.py +++ b/openpype/modules/ftrack/plugins/publish/collect_username.py @@ -13,7 +13,7 @@ import os import pyblish.api -class CollectUsername(pyblish.api.ContextPlugin): +class CollectUsernameForWebpublish(pyblish.api.ContextPlugin): """ Translates user email to Ftrack username. @@ -32,10 +32,8 @@ class CollectUsername(pyblish.api.ContextPlugin): hosts = ["webpublisher", "photoshop"] targets = ["remotepublish", "filespublish", "tvpaint_worker"] - _context = None - def process(self, context): - self.log.info("CollectUsername") + self.log.info("{}".format(self.__class__.__name__)) os.environ["FTRACK_API_USER"] = os.environ["FTRACK_BOT_API_USER"] os.environ["FTRACK_API_KEY"] = os.environ["FTRACK_BOT_API_KEY"] From 8e6b0567c967008b0b64151eeceed23a426ed073 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 1 Sep 2022 13:54:29 +0200 Subject: [PATCH 063/102] skip 'WEBPUBLISH_OPENPYPE_USERNAME' usage --- .../modules/ftrack/plugins/publish/collect_username.py | 1 - openpype/plugins/publish/extract_burnin.py | 7 ------- 2 files changed, 8 deletions(-) diff --git a/openpype/modules/ftrack/plugins/publish/collect_username.py b/openpype/modules/ftrack/plugins/publish/collect_username.py index 0e232bf83e..ab1f7d8d5d 100644 --- a/openpype/modules/ftrack/plugins/publish/collect_username.py +++ b/openpype/modules/ftrack/plugins/publish/collect_username.py @@ -65,5 +65,4 @@ class CollectUsernameForWebpublish(pyblish.api.ContextPlugin): burnin_name = username if '@' in burnin_name: burnin_name = burnin_name[:burnin_name.index('@')] - os.environ["WEBPUBLISH_OPENPYPE_USERNAME"] = burnin_name context.data["user"] = burnin_name diff --git a/openpype/plugins/publish/extract_burnin.py b/openpype/plugins/publish/extract_burnin.py index 36a28beb5d..c8b2b73874 100644 --- a/openpype/plugins/publish/extract_burnin.py +++ b/openpype/plugins/publish/extract_burnin.py @@ -488,13 +488,6 @@ class ExtractBurnin(openpype.api.Extractor): "frame_end_handle": frame_end_handle } - # use explicit username for webpublishes as rewriting - # OPENPYPE_USERNAME might have side effects - webpublish_user_name = os.environ.get("WEBPUBLISH_OPENPYPE_USERNAME") - if webpublish_user_name: - burnin_data["user"] = webpublish_user_name - burnin_data["username"] = webpublish_user_name - self.log.debug( "Basic burnin_data: {}".format(json.dumps(burnin_data, indent=4)) ) From 3ac5305f65b17083b126eeb76675064311bf4f4a Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 1 Sep 2022 13:54:58 +0200 Subject: [PATCH 064/102] better user query --- .../modules/ftrack/plugins/publish/collect_username.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/openpype/modules/ftrack/plugins/publish/collect_username.py b/openpype/modules/ftrack/plugins/publish/collect_username.py index ab1f7d8d5d..798f3960a8 100644 --- a/openpype/modules/ftrack/plugins/publish/collect_username.py +++ b/openpype/modules/ftrack/plugins/publish/collect_username.py @@ -1,5 +1,8 @@ """Loads publishing context from json and continues in publish process. +Should run before 'CollectAnatomyContextData' so the user on context is +changed before it's stored to context anatomy data or instance anatomy data. + Requires: anatomy -> context["anatomy"] *(pyblish.api.CollectorOrder - 0.11) @@ -52,12 +55,14 @@ class CollectUsernameForWebpublish(pyblish.api.ContextPlugin): return session = ftrack_api.Session(auto_connect_event_hub=False) - user = session.query("User where email like '{}'".format(user_email)) + user = session.query( + "User where email like '{}'".format(user_email) + ).first() if not user: raise ValueError( "Couldn't find user with {} email".format(user_email)) - user = user[0] + username = user.get("username") self.log.debug("Resolved ftrack username:: {}".format(username)) os.environ["FTRACK_API_USER"] = username From 3c6b999d544d65a8d31cc37ae54c731ff098f73a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Albert?= Date: Thu, 1 Sep 2022 17:22:14 +0200 Subject: [PATCH 065/102] Kitsu - sync_all_project - add list ignore_projects --- openpype/modules/kitsu/utils/update_op_with_zou.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/openpype/modules/kitsu/utils/update_op_with_zou.py b/openpype/modules/kitsu/utils/update_op_with_zou.py index 8d65591c0b..f013251bb1 100644 --- a/openpype/modules/kitsu/utils/update_op_with_zou.py +++ b/openpype/modules/kitsu/utils/update_op_with_zou.py @@ -318,13 +318,13 @@ def write_project_to_op(project: dict, dbcon: AvalonMongoDB) -> UpdateOne: ) -def sync_all_projects(login: str, password: str): +def sync_all_projects(login: str, password: str, ignore_projects: list = []): """Update all OP projects in DB with Zou data. Args: login (str): Kitsu user login password (str): Kitsu user password - + ignore_projects (list): List of unsynced project names Raises: gazu.exception.AuthFailedException: Wrong user login and/or password """ @@ -340,7 +340,8 @@ def sync_all_projects(login: str, password: str): dbcon.install() all_projects = gazu.project.all_open_projects() for project in all_projects: - sync_project_from_kitsu(dbcon, project) + if project["name"] not in ignore_projects: + sync_project_from_kitsu(dbcon, project) def sync_project_from_kitsu(dbcon: AvalonMongoDB, project: dict): From 4bdd18cb817bbc58c1143e0e02442a9346ce9a1e Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Fri, 2 Sep 2022 11:50:14 +0200 Subject: [PATCH 066/102] Use DeadlineKeyValueVar for EnvironmentKeyValue on Job Info - To improve readability of code that sets the values --- .../deadline/abstract_submit_deadline.py | 58 +++++++++++++++---- .../publish/submit_aftereffects_deadline.py | 11 ++-- .../publish/submit_harmony_deadline.py | 10 ++-- .../plugins/publish/submit_maya_deadline.py | 7 +-- 4 files changed, 59 insertions(+), 27 deletions(-) diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py index 35b114da95..beb1cd0fae 100644 --- a/openpype/modules/deadline/abstract_submit_deadline.py +++ b/openpype/modules/deadline/abstract_submit_deadline.py @@ -67,6 +67,43 @@ def requests_get(*args, **kwargs): return requests.get(*args, **kwargs) +class DeadlineKeyValueVar(dict): + """ + + Serializes dictionary key values as "{key}={value}" like Deadline uses + for EnvironmentKeyValue. + + As an example: + EnvironmentKeyValue0="A_KEY=VALUE_A" + EnvironmentKeyValue1="OTHER_KEY=VALUE_B" + + The keys are serialized in alphabetical order (sorted). + + Example: + >>> var = DeadlineKeyValueVar("EnvironmentKeyValue") + >>> var["my_var"] = "hello" + >>> var["my_other_var"] = "hello2" + >>> var.serialize() + + + """ + def __init__(self, key): + super(DeadlineKeyValueVar, self).__init__() + self.__key = key + + def serialize(self): + key = self.__key + + # Allow custom location for index in serialized string + if "{}" not in key: + key = key + "{}" + + return { + key.format(index): "{}={}".format(var_key, var_value) + for index, (var_key, var_value) in enumerate(sorted(self.items())) + } + + class DeadlineIndexedVar(dict): """ @@ -80,15 +117,9 @@ class DeadlineIndexedVar(dict): """ def __init__(self, key): + super(DeadlineIndexedVar, self).__init__() self.__key = key - def next_available_index(self): - # Add as first unused entry - i = 0 - while i in self.keys(): - i += 1 - return i - def serialize(self): key = self.__key @@ -100,6 +131,13 @@ class DeadlineIndexedVar(dict): key.format(index): value for index, value in sorted(self.items()) } + def next_available_index(self): + # Add as first unused entry + i = 0 + while i in self.keys(): + i += 1 + return i + def update(self, data): # Force the integer key check for key, value in data.items(): @@ -271,7 +309,7 @@ class DeadlineJobInfo(object): # Environment # ---------------------------------------------- - EnvironmentKeyValue = attr.ib(factory=partial(DeadlineIndexedVar, + EnvironmentKeyValue = attr.ib(factory=partial(DeadlineKeyValueVar, "EnvironmentKeyValue")) IncludeEnvironment = attr.ib(default=None) # Default: false @@ -281,7 +319,7 @@ class DeadlineJobInfo(object): # Job Extra Info # ---------------------------------------------- ExtraInfo = attr.ib(factory=partial(DeadlineIndexedVar, "ExtraInfo")) - ExtraInfoKeyValue = attr.ib(factory=partial(DeadlineIndexedVar, + ExtraInfoKeyValue = attr.ib(factory=partial(DeadlineKeyValueVar, "ExtraInfoKeyValue")) # Task Extra Info Names @@ -326,7 +364,7 @@ class DeadlineJobInfo(object): """ def filter_data(a, v): - if isinstance(v, DeadlineIndexedVar): + if isinstance(v, (DeadlineIndexedVar, DeadlineKeyValueVar)): return False if v is None: return False diff --git a/openpype/modules/deadline/plugins/publish/submit_aftereffects_deadline.py b/openpype/modules/deadline/plugins/publish/submit_aftereffects_deadline.py index 1d68793d53..55acd92043 100644 --- a/openpype/modules/deadline/plugins/publish/submit_aftereffects_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_aftereffects_deadline.py @@ -92,13 +92,12 @@ class AfterEffectsSubmitDeadline( environment = dict({key: os.environ[key] for key in keys if key in os.environ}, **legacy_io.Session) for key in keys: - val = environment.get(key) - if val: - dln_job_info.EnvironmentKeyValue = "{key}={value}".format( - key=key, - value=val) + value = environment.get(key) + if value: + dln_job_info.EnvironmentKeyValue[key] = value + # to recognize job from PYPE for turning Event On/Off - dln_job_info.EnvironmentKeyValue = "OPENPYPE_RENDER_JOB=1" + dln_job_info.EnvironmentKeyValue["OPENPYPE_RENDER_JOB"] = "1" return dln_job_info diff --git a/openpype/modules/deadline/plugins/publish/submit_harmony_deadline.py b/openpype/modules/deadline/plugins/publish/submit_harmony_deadline.py index 3f9c09b592..6327143623 100644 --- a/openpype/modules/deadline/plugins/publish/submit_harmony_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_harmony_deadline.py @@ -284,14 +284,12 @@ class HarmonySubmitDeadline( environment = dict({key: os.environ[key] for key in keys if key in os.environ}, **legacy_io.Session) for key in keys: - val = environment.get(key) - if val: - job_info.EnvironmentKeyValue = "{key}={value}".format( - key=key, - value=val) + value = environment.get(key) + if value: + job_info.EnvironmentKeyValue[key] = value # to recognize job from PYPE for turning Event On/Off - job_info.EnvironmentKeyValue = "OPENPYPE_RENDER_JOB=1" + job_info.EnvironmentKeyValue["OPENPYPE_RENDER_JOB"] = "1" return job_info diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 9692b136e9..ad46feea03 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -137,8 +137,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): for key, value in environment.items(): if not value: continue - job_info.EnvironmentKeyValue = "{key}={value}".format(key=key, - value=value) + job_info.EnvironmentKeyValue[key] = value # Adding file dependencies. if self.asset_dependencies: @@ -538,9 +537,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): for key, value in envs.items(): if not value: continue - - job_info.EnvironmentKeyValue = "{key}={value}".format(key=key, - value=value) + job_info.EnvironmentKeyValue[key] = value plugin_info.update({ "Version": "3.6", From 1e87c9d6d2c7338f8e53e8a06d9f1983056797b6 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Fri, 2 Sep 2022 12:02:39 +0200 Subject: [PATCH 067/102] Use DeadlineIndexedVar `__iadd__` functionality --- .../plugins/publish/submit_aftereffects_deadline.py | 4 ++-- .../deadline/plugins/publish/submit_maya_deadline.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_aftereffects_deadline.py b/openpype/modules/deadline/plugins/publish/submit_aftereffects_deadline.py index 55acd92043..0c1ffa6bd7 100644 --- a/openpype/modules/deadline/plugins/publish/submit_aftereffects_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_aftereffects_deadline.py @@ -67,9 +67,9 @@ class AfterEffectsSubmitDeadline( dln_job_info.Group = self.group dln_job_info.Department = self.department dln_job_info.ChunkSize = self.chunk_size - dln_job_info.OutputFilename = \ + dln_job_info.OutputFilename += \ os.path.basename(self._instance.data["expectedFiles"][0]) - dln_job_info.OutputDirectory = \ + dln_job_info.OutputDirectory += \ os.path.dirname(self._instance.data["expectedFiles"][0]) dln_job_info.JobDelay = "00:00:00" diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index ad46feea03..6b08f9894d 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -144,14 +144,14 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): dependencies = instance.context.data["fileDependencies"] dependencies.append(context.data["currentFile"]) for dependency in dependencies: - job_info.AssetDependency = dependency + job_info.AssetDependency += dependency # Add list of expected files to job # --------------------------------- exp = instance.data.get("expectedFiles") for filepath in self._iter_expected_files(exp): - job_info.OutputDirectory = os.path.dirname(filepath) - job_info.OutputFilename = os.path.basename(filepath) + job_info.OutputDirectory += os.path.dirname(filepath) + job_info.OutputFilename += os.path.basename(filepath) return job_info @@ -443,7 +443,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): if self.asset_dependencies: # Asset dependency to wait for at least the scene file to sync. - job_info.AssetDependency = self.scene_path + job_info.AssetDependency += self.scene_path # Get layer prefix render_products = self._instance.data["renderProducts"] From 2c01cb806d68aa04a733e16d4cfd1abb15f438fe Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Fri, 2 Sep 2022 12:03:34 +0200 Subject: [PATCH 068/102] Remove backwards compatibility for append functionality in old style vars --- openpype/modules/deadline/abstract_submit_deadline.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py index beb1cd0fae..f698b7688e 100644 --- a/openpype/modules/deadline/abstract_submit_deadline.py +++ b/openpype/modules/deadline/abstract_submit_deadline.py @@ -393,16 +393,6 @@ class DeadlineJobInfo(object): for key, value in data.items(): setattr(self, key, value) - def __setattr__(self, key, value): - # Backwards compatibility: Allow appending to index vars by setting - # it on Job Info directly like: JobInfo.OutputFilename = filename - existing = getattr(self, key, None) - if isinstance(existing, DeadlineIndexedVar): - existing += value - return - - object.__setattr__(self, key, value) - @six.add_metaclass(AbstractMetaInstancePlugin) class AbstractSubmitDeadline(pyblish.api.InstancePlugin): From 47164b36effa0f4986ccf15aa3a4967ccb014e26 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Fri, 2 Sep 2022 12:11:35 +0200 Subject: [PATCH 069/102] Be more explicit about what keys to include from Session This way it matches more with logic of other host submitters (e.g. AfterEffects + Harmony) --- .../deadline/plugins/publish/submit_maya_deadline.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 6b08f9894d..bb48fe6902 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -130,15 +130,16 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): environment = dict({key: os.environ[key] for key in keys if key in os.environ}, **legacy_io.Session) - # to recognize job from PYPE for turning Event On/Off - environment["OPENPYPE_RENDER_JOB"] = "1" - environment["OPENPYPE_LOG_NO_COLORS"] = "1" - - for key, value in environment.items(): + for key in keys: + value = environment.get(key) if not value: continue job_info.EnvironmentKeyValue[key] = value + # to recognize job from PYPE for turning Event On/Off + job_info.EnvironmentKeyValue["OPENPYPE_RENDER_JOB"] = "1" + job_info.EnvironmentKeyValue["OPENPYPE_LOG_NO_COLORS"] = "1" + # Adding file dependencies. if self.asset_dependencies: dependencies = instance.context.data["fileDependencies"] From 37371936cf293200b74b7d5ee5381aac8a0551ba Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 2 Sep 2022 14:08:40 +0200 Subject: [PATCH 070/102] safer task data --- .../plugins/publish/integrate_slack_api.py | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/openpype/modules/slack/plugins/publish/integrate_slack_api.py b/openpype/modules/slack/plugins/publish/integrate_slack_api.py index 4a8e9f773f..643e55915b 100644 --- a/openpype/modules/slack/plugins/publish/integrate_slack_api.py +++ b/openpype/modules/slack/plugins/publish/integrate_slack_api.py @@ -112,13 +112,19 @@ class IntegrateSlackAPI(pyblish.api.InstancePlugin): if review_path: fill_pairs.append(("review_filepath", review_path)) - task_data = instance.data.get("task") - if not task_data: - task_data = fill_data.get("task") - for key, value in task_data.items(): - fill_key = "task[{}]".format(key) - fill_pairs.append((fill_key, value)) - fill_pairs.append(("task", task_data["name"])) + task_data = fill_data.get("task") + if task_data: + if ( + "{task}" in message_templ + or "{Task}" in message_templ + or "{TASK}" in message_templ + ): + fill_pairs.append(("task", task_data["name"])) + + else: + for key, value in task_data.items(): + fill_key = "task[{}]".format(key) + fill_pairs.append((fill_key, value)) self.log.debug("fill_pairs ::{}".format(fill_pairs)) multiple_case_variants = prepare_template_data(fill_pairs) From 58fd5a1b097bd7d7e71f004b3fff8529296b1102 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Albert?= Date: Fri, 2 Sep 2022 16:32:52 +0200 Subject: [PATCH 071/102] Make to Optional Arguments --- openpype/modules/kitsu/utils/update_op_with_zou.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/kitsu/utils/update_op_with_zou.py b/openpype/modules/kitsu/utils/update_op_with_zou.py index f013251bb1..199c59053b 100644 --- a/openpype/modules/kitsu/utils/update_op_with_zou.py +++ b/openpype/modules/kitsu/utils/update_op_with_zou.py @@ -318,7 +318,7 @@ def write_project_to_op(project: dict, dbcon: AvalonMongoDB) -> UpdateOne: ) -def sync_all_projects(login: str, password: str, ignore_projects: list = []): +def sync_all_projects(login: str, password: str, ignore_projects=[]): """Update all OP projects in DB with Zou data. Args: From 88e4798b535c47cefce3dc2a1ed9aacf60dd0f68 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Fri, 2 Sep 2022 19:50:56 +0200 Subject: [PATCH 072/102] Remove old type hint --- .../modules/deadline/plugins/publish/submit_maya_deadline.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index bb48fe6902..68d55fef5d 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -641,7 +641,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): return result def _patch_workfile(self): - # type: (str, dict) -> [str, None] """Patch Maya scene. This will take list of patches (lines to add) and apply them to From 5645bcb353b13b1711ba67e0a3b394b273e7cef3 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Sat, 3 Sep 2022 13:17:10 +0200 Subject: [PATCH 073/102] Use custom plugin info per type of plugin submission --- .../plugins/publish/submit_maya_deadline.py | 127 +++++++++--------- 1 file changed, 66 insertions(+), 61 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 68d55fef5d..2a41d92efd 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -49,6 +49,30 @@ class MayaPluginInfo: RenderSetupIncludeLights = attr.ib(default=None) # Include all lights flag +@attr.s +class PythonPluginInfo: + ScriptFile = attr.ib() + Version = attr.ib(default="3.6") + Arguments = attr.ib(default=None) + SingleFrameOnly = attr.ib(default=None) + + +@attr.s +class VRayPluginInfo: + InputFilename = attr.ib(default=None) # Input + SeparateFilesPerFrame = attr.ib(default=None) + VRayEngine = attr.ib(default="V-Ray") + Width = attr.ib(default=None) + Height = attr.ib(default=None) # Mandatory for Deadline + OutputFilePath = attr.ib(default=True) + OutputFileName = attr.ib(default=None) # Render only this layer + + +@attr.s +class ArnoldPluginInfo: + ArnoldFile = attr.ib(default=None) + + class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): label = "Submit Render to Deadline" @@ -479,26 +503,19 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): def _get_vray_export_payload(self, data): job_info = copy.deepcopy(self.job_info) - job_info.Name = self._job_info_label("Export") # Get V-Ray settings info to compute output path - vray_settings = cmds.ls(type="VRaySettingsNode") - node = vray_settings[0] - template = cmds.getAttr("{}.vrscene_filename".format(node)) - scene, _ = os.path.splitext(data["filename"]) - first_file = self.format_vray_output_filename(scene, template) - first_file = "{}/{}".format(data["workspace"], first_file) - output = os.path.dirname(first_file) + vray_scene = self.format_vray_output_filename() plugin_info = { "Renderer": "vray", "SkipExistingFrames": True, "UseLegacyRenderLayers": True, - "OutputFilePath": output + "OutputFilePath": os.path.dirname(vray_scene) } - return job_info, plugin_info + return job_info, attr.asdict(plugin_info) def _get_arnold_export_payload(self, data): @@ -515,8 +532,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): script = os.path.normpath(module_path) job_info = copy.deepcopy(self.job_info) - plugin_info = copy.deepcopy(self.plugin_info) - job_info.Name = self._job_info_label("Export") # Force a single frame Python job @@ -540,14 +555,14 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): continue job_info.EnvironmentKeyValue[key] = value - plugin_info.update({ - "Version": "3.6", - "ScriptFile": script, - "Arguments": "", - "SingleFrameOnly": "True", - }) + plugin_info = PythonPluginInfo( + ScriptFile=script, + Version="3.6", + Arguments="", + SingleFrameOnly="True" + ) - return job_info, plugin_info + return job_info, attr.asdict(plugin_info) def _get_vray_render_payload(self, data): @@ -558,27 +573,17 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): job_info.OverrideTaskExtraInfoNames = False # Plugin Info - vray_settings = cmds.ls(type="VRaySettingsNode") - node = vray_settings[0] - template = cmds.getAttr("{}.vrscene_filename".format(node)) - # "vrayscene//_/" + plugin_info = VRayPluginInfo( + InputFilename=self.format_vray_output_filename(), + SeparateFilesPerFrame=False, + VRayEngine="V-Ray", + Width=self._instance.data["resolutionWidth"], + Height=self._instance.data["resolutionHeight"], + OutputFilePath=job_info.OutputDirectory[0], + OutputFileName=job_info.OutputFilename[0] + ) - scene, _ = os.path.splitext(self.scene_path) - first_file = self.format_vray_output_filename(scene, template) - first_file = "{}/{}".format(data["workspace"], first_file) - - plugin_info = { - "InputFilename": first_file, - "SeparateFilesPerFrame": True, - "VRayEngine": "V-Ray", - - "Width": self._instance.data["resolutionWidth"], - "Height": self._instance.data["resolutionHeight"], - "OutputFilePath": job_info.OutputDirectory[0], - "OutputFileName": job_info.OutputFilename[0] - } - - return job_info, plugin_info + return job_info, attr.asdict(plugin_info) def _get_arnold_render_payload(self, data): @@ -590,55 +595,55 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): # Plugin Info ass_file, _ = os.path.splitext(data["output_filename_0"]) - first_file = ass_file + ".ass" - plugin_info = { - "ArnoldFile": first_file, - } + ass_filepath = ass_file + ".ass" - return job_info, plugin_info + plugin_info = ArnoldPluginInfo( + ArnoldFile=ass_filepath + ) - def format_vray_output_filename(self, filename, template, dir=False): + return job_info, attr.asdict(plugin_info) + + def format_vray_output_filename(self): """Format the expected output file of the Export job. Example: /_/ - "shot010_v006/shot010_v006_CHARS/CHARS" - - Args: - instance: - filename(str): - dir(bool): - + "shot010_v006/shot010_v006_CHARS/CHARS_0001.vrscene" Returns: str """ + + # "vrayscene//_/" + vray_settings = cmds.ls(type="VRaySettingsNode") + node = vray_settings[0] + template = cmds.getAttr("{}.vrscene_filename".format(node)) + scene, _ = os.path.splitext(self.scene_path) + def smart_replace(string, key_values): new_string = string for key, value in key_values.items(): new_string = new_string.replace(key, value) return new_string - # Ensure filename has no extension - file_name, _ = os.path.splitext(filename) + # Get workfile scene path without extension to format vrscene_filename + scene_filename = os.path.basename(self.scene_path) + scene_filename_no_ext, _ = os.path.splitext(scene_filename) layer = self._instance.data['setMembers'] # Reformat without tokens output_path = smart_replace( template, - {"": file_name, + {"": scene_filename_no_ext, "": layer}) - if dir: - return output_path.replace("\\", "/") - start_frame = int(self._instance.data["frameStartHandle"]) + workspace = self._instance.context.data["workspace"] filename_zero = "{}_{:04d}.vrscene".format(output_path, start_frame) + filepath_zero = os.path.join(workspace, filename_zero) - result = filename_zero.replace("\\", "/") - - return result + return filepath_zero.replace("\\", "/") def _patch_workfile(self): """Patch Maya scene. From 507dac4aa9f50e8978a841067262ce33e77cf5e0 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Sat, 3 Sep 2022 15:06:24 +0200 Subject: [PATCH 074/102] Ensure integer math for _format_tiles See #3758 --- .../plugins/publish/submit_maya_deadline.py | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 2a41d92efd..7c486b7c34 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -754,14 +754,21 @@ def _format_tiles( used for assembler configuration. """ - tile = 0 + # Math used requires integers for correct output - as such + # we ensure our inputs are correct. + assert type(tiles_x) is int, "tiles_x must be an integer" + assert type(tiles_y) is int, "tiles_y must be an integer" + assert type(width) is int, "width must be an integer" + assert type(height) is int, "height must be an integer" + out = {"JobInfo": {}, "PluginInfo": {}} cfg = OrderedDict() - w_space = width / tiles_x - h_space = height / tiles_y + w_space = width // tiles_x + h_space = height // tiles_y cfg["TilesCropped"] = "False" + tile = 0 for tile_x in range(1, tiles_x + 1): for tile_y in reversed(range(1, tiles_y + 1)): tile_prefix = "_tile_{}x{}_{}x{}_".format( @@ -769,10 +776,10 @@ def _format_tiles( tiles_x, tiles_y ) - top = int(height - (tile_y * h_space)) - bottom = int(height - ((tile_y - 1) * h_space) - 1) - left = int((tile_x - 1) * w_space) - right = int((tile_x * w_space) - 1) + top = height - (tile_y * h_space) + bottom = height - ((tile_y - 1) * h_space) - 1 + left = (tile_x - 1) * w_space + right = (tile_x * w_space) - 1 # Job Info new_filename = "{}/{}{}".format( From f54f4cf99c824e927e5cbe24c7a02ec0e4a4fc62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Albert?= Date: Sun, 4 Sep 2022 22:43:25 +0200 Subject: [PATCH 075/102] Kitsu : Modification default value for Ignore_projects --- openpype/modules/kitsu/utils/update_op_with_zou.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/openpype/modules/kitsu/utils/update_op_with_zou.py b/openpype/modules/kitsu/utils/update_op_with_zou.py index 199c59053b..26cd125e15 100644 --- a/openpype/modules/kitsu/utils/update_op_with_zou.py +++ b/openpype/modules/kitsu/utils/update_op_with_zou.py @@ -318,7 +318,7 @@ def write_project_to_op(project: dict, dbcon: AvalonMongoDB) -> UpdateOne: ) -def sync_all_projects(login: str, password: str, ignore_projects=[]): +def sync_all_projects(login: str, password: str, ignore_projects: list = None): """Update all OP projects in DB with Zou data. Args: @@ -340,8 +340,9 @@ def sync_all_projects(login: str, password: str, ignore_projects=[]): dbcon.install() all_projects = gazu.project.all_open_projects() for project in all_projects: - if project["name"] not in ignore_projects: - sync_project_from_kitsu(dbcon, project) + if ignore_projects and project["name"] in ignore_projects: + continue + sync_project_from_kitsu(dbcon, project) def sync_project_from_kitsu(dbcon: AvalonMongoDB, project: dict): From fb48faf386c8d53c737ac9a6a00287730c934217 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Albert?= Date: Sun, 4 Sep 2022 22:49:07 +0200 Subject: [PATCH 076/102] Kitsu : Ignore_projects - minor fix - indent --- openpype/modules/kitsu/utils/update_op_with_zou.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/kitsu/utils/update_op_with_zou.py b/openpype/modules/kitsu/utils/update_op_with_zou.py index 26cd125e15..55a7bdc51d 100644 --- a/openpype/modules/kitsu/utils/update_op_with_zou.py +++ b/openpype/modules/kitsu/utils/update_op_with_zou.py @@ -341,7 +341,7 @@ def sync_all_projects(login: str, password: str, ignore_projects: list = None): all_projects = gazu.project.all_open_projects() for project in all_projects: if ignore_projects and project["name"] in ignore_projects: - continue + continue sync_project_from_kitsu(dbcon, project) From 5908f49b685a981d08d7ea1ff841d567018a7e76 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 6 Sep 2022 10:56:01 +0200 Subject: [PATCH 077/102] updating README file --- README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b8c04f8b49..a2f442b640 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ It can be built and ran on all common platforms. We develop and test on the foll - **Linux** - **Ubuntu** 20.04 LTS - **Centos** 7 -- **Mac OSX** +- **Mac OSX** - **10.15** Catalina - **11.1** Big Sur (using Rosetta2) @@ -287,6 +287,14 @@ To run tests, execute `.\tools\run_tests(.ps1|.sh)`. **Note that it needs existing virtual environment.** + +Developer tools +------------- + +In case you wish to add your own tools to `.\tools` folder without git tracking, it is possible by adding it with `dev_*` suffix (`dev_clear_pyc(.ps1|.sh)`). + + + ## Contributors ✨ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)): From a0e241b02fbfdee18b8ded65af89eedb343793d2 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 6 Sep 2022 10:58:22 +0200 Subject: [PATCH 078/102] README fix --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a2f442b640..a3d3cf1dbb 100644 --- a/README.md +++ b/README.md @@ -291,7 +291,7 @@ To run tests, execute `.\tools\run_tests(.ps1|.sh)`. Developer tools ------------- -In case you wish to add your own tools to `.\tools` folder without git tracking, it is possible by adding it with `dev_*` suffix (`dev_clear_pyc(.ps1|.sh)`). +In case you wish to add your own tools to `.\tools` folder without git tracking, it is possible by adding it with `dev_*` suffix (example: `dev_clear_pyc(.ps1|.sh)`). From 70a17a8876541de84d98bd8f502421827fd78751 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 6 Sep 2022 11:39:58 +0200 Subject: [PATCH 079/102] hiero: instances detection - timeline no need to retime --- openpype/hosts/hiero/plugins/publish/precollect_instances.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/openpype/hosts/hiero/plugins/publish/precollect_instances.py b/openpype/hosts/hiero/plugins/publish/precollect_instances.py index 0c7dbc1f22..84f2927fc7 100644 --- a/openpype/hosts/hiero/plugins/publish/precollect_instances.py +++ b/openpype/hosts/hiero/plugins/publish/precollect_instances.py @@ -318,10 +318,9 @@ class PrecollectInstances(pyblish.api.ContextPlugin): @staticmethod def create_otio_time_range_from_timeline_item_data(track_item): - speed = track_item.playbackSpeed() timeline = phiero.get_current_sequence() frame_start = int(track_item.timelineIn()) - frame_duration = int((track_item.duration() - 1) / speed) + frame_duration = int(track_item.duration()) fps = timeline.framerate().toFloat() return hiero_export.create_otio_time_range( From 1ed5ef7dcdf9cd2183f009c541e4c52f5374075d Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 6 Sep 2022 13:16:01 +0200 Subject: [PATCH 080/102] OP-3863 - attempt to speed up ExtractImage If workfile has a large number of layers (hundreds or thousands), ExtractImage wasn't too efficient. It was hiding/showing layers too many times. Current logic is to hide all, show only publishable layers of instance, save to image, hide them again. get_layers replaced by argument if possible. --- openpype/hosts/photoshop/api/lib.py | 11 +- openpype/hosts/photoshop/api/ws_stub.py | 19 ++- .../plugins/publish/extract_image.py | 120 +++++++++++------- 3 files changed, 97 insertions(+), 53 deletions(-) diff --git a/openpype/hosts/photoshop/api/lib.py b/openpype/hosts/photoshop/api/lib.py index 73a546604f..221b4314e6 100644 --- a/openpype/hosts/photoshop/api/lib.py +++ b/openpype/hosts/photoshop/api/lib.py @@ -64,10 +64,15 @@ def maintained_selection(): @contextlib.contextmanager -def maintained_visibility(): - """Maintain visibility during context.""" +def maintained_visibility(layers=None): + """Maintain visibility during context. + + Args: + layers (list) of PSItem (used for caching) + """ visibility = {} - layers = stub().get_layers() + if not layers: + layers = stub().get_layers() for layer in layers: visibility[layer.id] = layer.visible try: diff --git a/openpype/hosts/photoshop/api/ws_stub.py b/openpype/hosts/photoshop/api/ws_stub.py index b49bf1c73f..2c4d0ad5fc 100644 --- a/openpype/hosts/photoshop/api/ws_stub.py +++ b/openpype/hosts/photoshop/api/ws_stub.py @@ -229,10 +229,11 @@ class PhotoshopServerStub: return self._get_layers_in_layers(parent_ids) - def get_layers_in_layers_ids(self, layers_ids): + def get_layers_in_layers_ids(self, layers_ids, layers=None): """Return all layers that belong to layers (might be groups). Args: + layers_ids layers : Returns: @@ -240,10 +241,13 @@ class PhotoshopServerStub: """ parent_ids = set(layers_ids) - return self._get_layers_in_layers(parent_ids) + return self._get_layers_in_layers(parent_ids, layers) - def _get_layers_in_layers(self, parent_ids): - all_layers = self.get_layers() + def _get_layers_in_layers(self, parent_ids, layers=None): + if not layers: + layers = self.get_layers() + + all_layers = layers ret = [] for layer in all_layers: @@ -394,14 +398,17 @@ class PhotoshopServerStub: self.hide_all_others_layers_ids(extract_ids) - def hide_all_others_layers_ids(self, extract_ids): + def hide_all_others_layers_ids(self, extract_ids, layers=None): """hides all layers that are not part of the list or that are not children of this list Args: extract_ids (list): list of integer that should be visible + layers (list) of PSItem (used for caching) """ - for layer in self.get_layers(): + if not layers: + layers = self.get_layers() + for layer in layers: if layer.visible and layer.id not in extract_ids: self.set_visible(layer.id, False) diff --git a/openpype/hosts/photoshop/plugins/publish/extract_image.py b/openpype/hosts/photoshop/plugins/publish/extract_image.py index a133e33409..7543af95bd 100644 --- a/openpype/hosts/photoshop/plugins/publish/extract_image.py +++ b/openpype/hosts/photoshop/plugins/publish/extract_image.py @@ -1,61 +1,93 @@ import os -import openpype.api +import pyblish.api + +from openpype.pipeline import publish from openpype.hosts.photoshop import api as photoshop -class ExtractImage(openpype.api.Extractor): - """Produce a flattened image file from instance - - This plug-in takes into account only the layers in the group. - """ +class ExtractImage(pyblish.api.ContextPlugin): + """Save scene before extraction.""" + order = publish.Extractor.order - 0.48 label = "Extract Image" hosts = ["photoshop"] + families = ["image", "background"] formats = ["png", "jpg"] - def process(self, instance): - staging_dir = self.staging_dir(instance) - self.log.info("Outputting image to {}".format(staging_dir)) - - # Perform extraction + def process(self, context): stub = photoshop.stub() - files = {} + hidden_layer_ids = set() + + all_layers = stub.get_layers() + for layer in all_layers: + if not layer.visible: + hidden_layer_ids.add(layer.id) + stub.hide_all_others_layers_ids([], layers=all_layers) + with photoshop.maintained_selection(): - self.log.info("Extracting %s" % str(list(instance))) - with photoshop.maintained_visibility(): - ids = set() - layer = instance.data.get("layer") - if layer: - ids.add(layer.id) - add_ids = instance.data.pop("ids", None) - if add_ids: - ids.update(set(add_ids)) - extract_ids = set([ll.id for ll in stub. - get_layers_in_layers_ids(ids)]) - stub.hide_all_others_layers_ids(extract_ids) + # self.log.info("Extracting %s" % str(list(instance))) + with photoshop.maintained_visibility(layers=all_layers): + for instance in context: + if instance.data["family"] not in self.families: + continue - file_basename = os.path.splitext( - stub.get_active_document_name() - )[0] - for extension in self.formats: - _filename = "{}.{}".format(file_basename, extension) - files[extension] = _filename + staging_dir = self.staging_dir(instance) + self.log.info("Outputting image to {}".format(staging_dir)) - full_filename = os.path.join(staging_dir, _filename) - stub.saveAs(full_filename, extension, True) - self.log.info(f"Extracted: {extension}") + # Perform extraction + files = {} + ids = set() + layer = instance.data.get("layer") + if layer: + ids.add(layer.id) + add_ids = instance.data.pop("ids", None) + if add_ids: + ids.update(set(add_ids)) + extract_ids = set([ll.id for ll in stub. + get_layers_in_layers_ids(ids, all_layers) + if ll.id not in hidden_layer_ids]) - representations = [] - for extension, filename in files.items(): - representations.append({ - "name": extension, - "ext": extension, - "files": filename, - "stagingDir": staging_dir - }) - instance.data["representations"] = representations - instance.data["stagingDir"] = staging_dir + for extracted_id in extract_ids: + stub.set_visible(extracted_id, True) - self.log.info(f"Extracted {instance} to {staging_dir}") + file_basename = os.path.splitext( + stub.get_active_document_name() + )[0] + for extension in self.formats: + _filename = "{}.{}".format(file_basename, + extension) + files[extension] = _filename + + full_filename = os.path.join(staging_dir, + _filename) + stub.saveAs(full_filename, extension, True) + self.log.info(f"Extracted: {extension}") + + representations = [] + for extension, filename in files.items(): + representations.append({ + "name": extension, + "ext": extension, + "files": filename, + "stagingDir": staging_dir + }) + instance.data["representations"] = representations + instance.data["stagingDir"] = staging_dir + + self.log.info(f"Extracted {instance} to {staging_dir}") + + for extracted_id in extract_ids: + stub.set_visible(extracted_id, False) + + def staging_dir(self, instance): + """Provide a temporary directory in which to store extracted files + + Upon calling this method the staging directory is stored inside + the instance.data['stagingDir'] + """ + + from openpype.pipeline.publish import get_instance_staging_dir + + return get_instance_staging_dir(instance) From 96f8c3b7b3a1cf524c2ed1a815092ef2f55a2e9b Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 6 Sep 2022 13:23:41 +0200 Subject: [PATCH 081/102] OP-3863 - fixed doc string --- .../hosts/photoshop/plugins/publish/extract_image.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/photoshop/plugins/publish/extract_image.py b/openpype/hosts/photoshop/plugins/publish/extract_image.py index 7543af95bd..5bdb3ef681 100644 --- a/openpype/hosts/photoshop/plugins/publish/extract_image.py +++ b/openpype/hosts/photoshop/plugins/publish/extract_image.py @@ -7,7 +7,15 @@ from openpype.hosts.photoshop import api as photoshop class ExtractImage(pyblish.api.ContextPlugin): - """Save scene before extraction.""" + """Extract all layers (groups) marked for publish. + + Usually publishable instance is created as a wrapper of layer(s). For each + publishable instance so many images as there is 'formats' is created. + + Logic tries to hide/unhide layers minimum times. + + Called once for all publishable instances. + """ order = publish.Extractor.order - 0.48 label = "Extract Image" @@ -27,7 +35,6 @@ class ExtractImage(pyblish.api.ContextPlugin): stub.hide_all_others_layers_ids([], layers=all_layers) with photoshop.maintained_selection(): - # self.log.info("Extracting %s" % str(list(instance))) with photoshop.maintained_visibility(layers=all_layers): for instance in context: if instance.data["family"] not in self.families: From c8efd0d67e5600a093d024786dd16717c9f3d16c Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Tue, 6 Sep 2022 17:07:46 +0200 Subject: [PATCH 082/102] removed f-string formatting for py2 compatibility --- openpype/hosts/resolve/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/openpype/hosts/resolve/utils.py b/openpype/hosts/resolve/utils.py index 382a7cf344..aa8ad3008b 100644 --- a/openpype/hosts/resolve/utils.py +++ b/openpype/hosts/resolve/utils.py @@ -17,7 +17,7 @@ def setup(env): # collect script dirs if us_env: - log.info(f"Utility Scripts Env: `{us_env}`") + log.info("Utility Scripts Env: `{}`".format(us_env)) us_paths = us_env.split( os.pathsep) + us_paths @@ -25,8 +25,8 @@ def setup(env): for path in us_paths: scripts.update({path: os.listdir(path)}) - log.info(f"Utility Scripts Dir: `{us_paths}`") - log.info(f"Utility Scripts: `{scripts}`") + log.info("Utility Scripts Dir: `{}`".format(us_paths)) + log.info("Utility Scripts: `{}`".format(scripts)) # make sure no script file is in folder for s in os.listdir(us_dir): @@ -44,7 +44,7 @@ def setup(env): # script in script list src = os.path.join(d, s) dst = os.path.join(us_dir, s) - log.info(f"Copying `{src}` to `{dst}`...") + log.info("Copying `{}` to `{}`...".format(src, dst)) if os.path.isdir(src): shutil.copytree( src, dst, symlinks=False, From de52a7d61b970b6df353130295568a2242b6b902 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Tue, 6 Sep 2022 17:21:32 +0200 Subject: [PATCH 083/102] last f-string removement --- openpype/hosts/resolve/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/hosts/resolve/utils.py b/openpype/hosts/resolve/utils.py index aa8ad3008b..d5c133bbf5 100644 --- a/openpype/hosts/resolve/utils.py +++ b/openpype/hosts/resolve/utils.py @@ -31,7 +31,7 @@ def setup(env): # make sure no script file is in folder for s in os.listdir(us_dir): path = os.path.join(us_dir, s) - log.info(f"Removing `{path}`...") + log.info("Removing `{}`...".format(path)) if os.path.isdir(path): shutil.rmtree(path, onerror=None) else: From 915cba631c42a822be8f71517d081a43a09cc177 Mon Sep 17 00:00:00 2001 From: OpenPype Date: Wed, 7 Sep 2022 04:24:55 +0000 Subject: [PATCH 084/102] [Automated] Bump version --- CHANGELOG.md | 18 +++++++++--------- openpype/version.py | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b35e89e96e..6754f1e2e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,22 +1,25 @@ # Changelog -## [3.14.2-nightly.2](https://github.com/pypeclub/OpenPype/tree/HEAD) +## [3.14.2-nightly.3](https://github.com/pypeclub/OpenPype/tree/HEAD) [Full Changelog](https://github.com/pypeclub/OpenPype/compare/3.14.1...HEAD) **🆕 New features** - Nuke: Build workfile by template [\#3763](https://github.com/pypeclub/OpenPype/pull/3763) -- Houdini: Publishing workfiles [\#3697](https://github.com/pypeclub/OpenPype/pull/3697) **🚀 Enhancements** +- Photoshop: attempt to speed up ExtractImage [\#3793](https://github.com/pypeclub/OpenPype/pull/3793) - SyncServer: Added cli commands for sync server [\#3765](https://github.com/pypeclub/OpenPype/pull/3765) +- Blender: Publisher collect workfile representation [\#3670](https://github.com/pypeclub/OpenPype/pull/3670) - Maya: move set render settings menu entry [\#3669](https://github.com/pypeclub/OpenPype/pull/3669) - Scene Inventory: Maya add actions to select from or to scene [\#3659](https://github.com/pypeclub/OpenPype/pull/3659) **🐛 Bug fixes** +- Resolve: Addon import is Python 2 compatible [\#3798](https://github.com/pypeclub/OpenPype/pull/3798) +- nuke: validate write node is not failing due wrong type [\#3780](https://github.com/pypeclub/OpenPype/pull/3780) - Fix - changed format of version string in pyproject.toml [\#3777](https://github.com/pypeclub/OpenPype/pull/3777) - Ftrack status fix typo prgoress -\> progress [\#3761](https://github.com/pypeclub/OpenPype/pull/3761) - Fix version resolution [\#3757](https://github.com/pypeclub/OpenPype/pull/3757) @@ -24,10 +27,14 @@ **🔀 Refactored code** +- Photoshop: Use new Extractor location [\#3789](https://github.com/pypeclub/OpenPype/pull/3789) +- Blender: Use new Extractor location [\#3787](https://github.com/pypeclub/OpenPype/pull/3787) - AfterEffects: Use new Extractor location [\#3784](https://github.com/pypeclub/OpenPype/pull/3784) - General: Remove unused teshost [\#3773](https://github.com/pypeclub/OpenPype/pull/3773) - General: Copied 'Extractor' plugin to publish pipeline [\#3771](https://github.com/pypeclub/OpenPype/pull/3771) +- General: Move queries of asset and representation links [\#3770](https://github.com/pypeclub/OpenPype/pull/3770) - General: Create project function moved to client code [\#3766](https://github.com/pypeclub/OpenPype/pull/3766) +- General: Move delivery logic to pipeline [\#3751](https://github.com/pypeclub/OpenPype/pull/3751) - General: Move hostdirname functionality into host [\#3749](https://github.com/pypeclub/OpenPype/pull/3749) - General: Move publish utils to pipeline [\#3745](https://github.com/pypeclub/OpenPype/pull/3745) - Houdini: Define houdini as addon [\#3735](https://github.com/pypeclub/OpenPype/pull/3735) @@ -80,7 +87,6 @@ **🔀 Refactored code** -- General: Move delivery logic to pipeline [\#3751](https://github.com/pypeclub/OpenPype/pull/3751) - General: Host addons cleanup [\#3744](https://github.com/pypeclub/OpenPype/pull/3744) - Webpublisher: Webpublisher is used as addon [\#3740](https://github.com/pypeclub/OpenPype/pull/3740) - Photoshop: Defined photoshop as addon [\#3736](https://github.com/pypeclub/OpenPype/pull/3736) @@ -116,25 +122,19 @@ - Ftrack: Set task status on farm publishing [\#3680](https://github.com/pypeclub/OpenPype/pull/3680) - Ftrack: Set task status on task creation in integrate hierarchy [\#3675](https://github.com/pypeclub/OpenPype/pull/3675) - Maya: Disable rendering of all lights for render instances submitted through Deadline. [\#3661](https://github.com/pypeclub/OpenPype/pull/3661) -- General: Optimized OCIO configs [\#3650](https://github.com/pypeclub/OpenPype/pull/3650) **🐛 Bug fixes** - General: Switch from hero version to versioned works [\#3691](https://github.com/pypeclub/OpenPype/pull/3691) - General: Fix finding of last version [\#3656](https://github.com/pypeclub/OpenPype/pull/3656) -- General: Extract Review can scale with pixel aspect ratio [\#3644](https://github.com/pypeclub/OpenPype/pull/3644) -- Maya: Refactor moved usage of CreateRender settings [\#3643](https://github.com/pypeclub/OpenPype/pull/3643) **🔀 Refactored code** - General: Use client projects getter [\#3673](https://github.com/pypeclub/OpenPype/pull/3673) -- Resolve: Match folder structure to other hosts [\#3653](https://github.com/pypeclub/OpenPype/pull/3653) -- Maya: Hosts as modules [\#3647](https://github.com/pypeclub/OpenPype/pull/3647) **Merged pull requests:** - Deadline: Global job pre load is not Pype 2 compatible [\#3666](https://github.com/pypeclub/OpenPype/pull/3666) -- Maya: Remove unused get current renderer logic [\#3645](https://github.com/pypeclub/OpenPype/pull/3645) ## [3.13.0](https://github.com/pypeclub/OpenPype/tree/3.13.0) (2022-08-09) diff --git a/openpype/version.py b/openpype/version.py index 26b03c37e5..c042ca2625 100644 --- a/openpype/version.py +++ b/openpype/version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- """Package declaring Pype version.""" -__version__ = "3.14.2-nightly.2" +__version__ = "3.14.2-nightly.3" From e1c17c71d30a38bcfda292393e425651c4c5a6d4 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 7 Sep 2022 12:11:06 +0200 Subject: [PATCH 085/102] fix variable name --- openpype/pipeline/template_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/pipeline/template_data.py b/openpype/pipeline/template_data.py index bab46a627d..627eba5c3d 100644 --- a/openpype/pipeline/template_data.py +++ b/openpype/pipeline/template_data.py @@ -53,7 +53,7 @@ def get_project_template_data(project_doc=None, project_name=None): project_name = project_doc["name"] if not project_doc: - project_code = get_project(project_name, fields=["data.code"]) + project_doc = get_project(project_name, fields=["data.code"]) project_code = project_doc.get("data", {}).get("code") return { From 509c209093d6fd971ec3631e662a8e34a31c5717 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 7 Sep 2022 12:40:34 +0200 Subject: [PATCH 086/102] fix status handling --- igniter/install_dialog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/igniter/install_dialog.py b/igniter/install_dialog.py index b09529f5c5..c7e9ef74c5 100644 --- a/igniter/install_dialog.py +++ b/igniter/install_dialog.py @@ -389,7 +389,7 @@ class InstallDialog(QtWidgets.QDialog): def _installation_finished(self): status = self._install_thread.result() - if status >= 0: + if status is not None and status >= 0: self._update_progress(100) QtWidgets.QApplication.processEvents() self.done(3) From d6ab41887a37e1bfb4e7f9a1f430daef93182faa Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 7 Sep 2022 13:41:30 +0200 Subject: [PATCH 087/102] added comment to code --- igniter/install_dialog.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/igniter/install_dialog.py b/igniter/install_dialog.py index c7e9ef74c5..65ddd58735 100644 --- a/igniter/install_dialog.py +++ b/igniter/install_dialog.py @@ -388,6 +388,9 @@ class InstallDialog(QtWidgets.QDialog): install_thread.start() def _installation_finished(self): + # TODO we should find out why status can be set to 'None'? + # - 'InstallThread.run' should handle all cases so not sure where + # that come from status = self._install_thread.result() if status is not None and status >= 0: self._update_progress(100) From 57aa1e6659ab9552c1980dbebcd8b64535469f39 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 7 Sep 2022 14:49:55 +0200 Subject: [PATCH 088/102] implementing not retimed working frame range and retimed handles switch --- openpype/hosts/flame/api/plugin.py | 10 ++++++ .../flame/plugins/create/create_shot_clip.py | 16 ++++++++++ .../publish/collect_timeline_instances.py | 4 +++ .../publish/extract_subset_resources.py | 32 +++++++++++++------ .../publish/collect_otio_frame_ranges.py | 6 ++++ 5 files changed, 58 insertions(+), 10 deletions(-) diff --git a/openpype/hosts/flame/api/plugin.py b/openpype/hosts/flame/api/plugin.py index efbabb6a55..145b1f0921 100644 --- a/openpype/hosts/flame/api/plugin.py +++ b/openpype/hosts/flame/api/plugin.py @@ -361,6 +361,8 @@ class PublishableClip: index_from_segment_default = False use_shot_name_default = False include_handles_default = False + retimed_handles_default = True + retimed_framerange_default = True def __init__(self, segment, **kwargs): self.rename_index = kwargs["rename_index"] @@ -496,6 +498,14 @@ class PublishableClip: "audio", {}).get("value") or False self.include_handles = self.ui_inputs.get( "includeHandles", {}).get("value") or self.include_handles_default + self.retimed_handles = ( + self.ui_inputs.get("retimedHandles", {}).get("value") + or self.retimed_handles_default + ) + self.retimed_framerange = ( + self.ui_inputs.get("retimedFramerange", {}).get("value") + or self.retimed_framerange_default + ) # build subset name from layer name if self.subset_name == "[ track name ]": diff --git a/openpype/hosts/flame/plugins/create/create_shot_clip.py b/openpype/hosts/flame/plugins/create/create_shot_clip.py index fa239ea420..b03a39a7ca 100644 --- a/openpype/hosts/flame/plugins/create/create_shot_clip.py +++ b/openpype/hosts/flame/plugins/create/create_shot_clip.py @@ -276,6 +276,22 @@ class CreateShotClip(opfapi.Creator): "target": "tag", "toolTip": "By default handles are excluded", # noqa "order": 3 + }, + "retimedHandles": { + "value": True, + "type": "QCheckBox", + "label": "Retimed handles", + "target": "tag", + "toolTip": "By default handles are retimed.", # noqa + "order": 4 + }, + "retimedFramerange": { + "value": True, + "type": "QCheckBox", + "label": "Retimed framerange", + "target": "tag", + "toolTip": "By default framerange is retimed.", # noqa + "order": 5 } } } diff --git a/openpype/hosts/flame/plugins/publish/collect_timeline_instances.py b/openpype/hosts/flame/plugins/publish/collect_timeline_instances.py index 992db62c75..d6ff13b059 100644 --- a/openpype/hosts/flame/plugins/publish/collect_timeline_instances.py +++ b/openpype/hosts/flame/plugins/publish/collect_timeline_instances.py @@ -131,6 +131,10 @@ class CollectTimelineInstances(pyblish.api.ContextPlugin): "fps": self.fps, "workfileFrameStart": workfile_start, "sourceFirstFrame": int(first_frame), + "notRetimedHandles": ( + not marker_data.get("retimedHandles")), + "notRetimedFramerange": ( + not marker_data.get("retimedFramerange")), "path": file_path, "flameAddTasks": self.add_tasks, "tasks": { diff --git a/openpype/hosts/flame/plugins/publish/extract_subset_resources.py b/openpype/hosts/flame/plugins/publish/extract_subset_resources.py index 3e1e8db986..1af6b00654 100644 --- a/openpype/hosts/flame/plugins/publish/extract_subset_resources.py +++ b/openpype/hosts/flame/plugins/publish/extract_subset_resources.py @@ -90,26 +90,38 @@ class ExtractSubsetResources(openpype.api.Extractor): handle_end = instance.data["handleEnd"] handles = max(handle_start, handle_end) include_handles = instance.data.get("includeHandles") + retimed_handles = instance.data.get("retimedHandles") # get media source range with handles source_start_handles = instance.data["sourceStartH"] source_end_handles = instance.data["sourceEndH"] - # retime if needed + + # retime if needed if r_speed != 1.0: - source_start_handles = ( - instance.data["sourceStart"] - r_handle_start) - source_end_handles = ( - source_start_handles - + (r_source_dur - 1) - + r_handle_start - + r_handle_end - ) + if retimed_handles: + # handles are retimed + source_start_handles = ( + instance.data["sourceStart"] - r_handle_start) + source_end_handles = ( + source_start_handles + + (r_source_dur - 1) + + r_handle_start + + r_handle_end + ) + else: + # handles are not retimed + source_end_handles = ( + source_start_handles + + (r_source_dur - 1) + + handle_start + + handle_end + ) # get frame range with handles for representation range frame_start_handle = frame_start - handle_start repre_frame_start = frame_start_handle if include_handles: - if r_speed == 1.0: + if r_speed == 1.0 or not retimed_handles: frame_start_handle = frame_start else: frame_start_handle = ( diff --git a/openpype/plugins/publish/collect_otio_frame_ranges.py b/openpype/plugins/publish/collect_otio_frame_ranges.py index 40e89e29bc..40a3fa6978 100644 --- a/openpype/plugins/publish/collect_otio_frame_ranges.py +++ b/openpype/plugins/publish/collect_otio_frame_ranges.py @@ -29,6 +29,7 @@ class CollectOtioFrameRanges(pyblish.api.InstancePlugin): # get basic variables otio_clip = instance.data["otioClip"] workfile_start = instance.data["workfileFrameStart"] + not_retime_framerange = instance.data.get("notRetimedFramerange") # get ranges otio_tl_range = otio_clip.range_in_parent() @@ -54,6 +55,11 @@ class CollectOtioFrameRanges(pyblish.api.InstancePlugin): frame_end = frame_start + otio.opentime.to_frames( otio_tl_range.duration, otio_tl_range.duration.rate) - 1 + # in case of retimed clip and frame range should not be retimed + if not_retime_framerange: + frame_end = frame_start + otio.opentime.to_frames( + otio_src_range.duration, otio_src_range.duration.rate) - 1 + data = { "frameStart": frame_start, "frameEnd": frame_end, From d2b3c80fb93acf26f9d61a6164dadf2941cd7930 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 7 Sep 2022 16:02:38 +0200 Subject: [PATCH 089/102] improving variable name --- openpype/plugins/publish/collect_otio_frame_ranges.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/plugins/publish/collect_otio_frame_ranges.py b/openpype/plugins/publish/collect_otio_frame_ranges.py index 40a3fa6978..cfb0318950 100644 --- a/openpype/plugins/publish/collect_otio_frame_ranges.py +++ b/openpype/plugins/publish/collect_otio_frame_ranges.py @@ -29,7 +29,7 @@ class CollectOtioFrameRanges(pyblish.api.InstancePlugin): # get basic variables otio_clip = instance.data["otioClip"] workfile_start = instance.data["workfileFrameStart"] - not_retime_framerange = instance.data.get("notRetimedFramerange") + workfile_source_duration = instance.data.get("notRetimedFramerange") # get ranges otio_tl_range = otio_clip.range_in_parent() @@ -56,7 +56,7 @@ class CollectOtioFrameRanges(pyblish.api.InstancePlugin): otio_tl_range.duration, otio_tl_range.duration.rate) - 1 # in case of retimed clip and frame range should not be retimed - if not_retime_framerange: + if workfile_source_duration: frame_end = frame_start + otio.opentime.to_frames( otio_src_range.duration, otio_src_range.duration.rate) - 1 From 5627c8ec627e4157e45eaeeb5a58e12216979f7c Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 8 Sep 2022 11:23:56 +0200 Subject: [PATCH 090/102] enabled pixmap scaling in tray --- openpype/tools/tray/pype_tray.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/openpype/tools/tray/pype_tray.py b/openpype/tools/tray/pype_tray.py index 85bc00ead6..c32a074fd1 100644 --- a/openpype/tools/tray/pype_tray.py +++ b/openpype/tools/tray/pype_tray.py @@ -778,6 +778,14 @@ def main(): if not app: app = QtWidgets.QApplication([]) + for attr_name in ( + "AA_EnableHighDpiScaling", + "AA_UseHighDpiPixmaps" + ): + attr = getattr(QtCore.Qt, attr_name, None) + if attr is not None: + app.setAttribute(attr) + starter = PypeTrayStarter(app) # TODO remove when pype.exe will have an icon From 4f3accee1ac84991543493a32768c1aa99672035 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 8 Sep 2022 11:34:33 +0200 Subject: [PATCH 091/102] change "hierarchy" key to "nodesHierarchy" in maya --- openpype/hosts/maya/plugins/publish/collect_assembly.py | 2 +- openpype/hosts/maya/plugins/publish/extract_assembly.py | 2 +- .../hosts/maya/plugins/publish/validate_assembly_transforms.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/collect_assembly.py b/openpype/hosts/maya/plugins/publish/collect_assembly.py index 1a65bf1fde..2aef9ab908 100644 --- a/openpype/hosts/maya/plugins/publish/collect_assembly.py +++ b/openpype/hosts/maya/plugins/publish/collect_assembly.py @@ -70,7 +70,7 @@ class CollectAssembly(pyblish.api.InstancePlugin): data[representation_id].append(instance_data) instance.data["scenedata"] = dict(data) - instance.data["hierarchy"] = list(set(hierarchy_nodes)) + instance.data["nodesHierarchy"] = list(set(hierarchy_nodes)) def get_file_rule(self, rule): return mel.eval('workspace -query -fileRuleEntry "{}"'.format(rule)) diff --git a/openpype/hosts/maya/plugins/publish/extract_assembly.py b/openpype/hosts/maya/plugins/publish/extract_assembly.py index 482930b76e..120805894e 100644 --- a/openpype/hosts/maya/plugins/publish/extract_assembly.py +++ b/openpype/hosts/maya/plugins/publish/extract_assembly.py @@ -33,7 +33,7 @@ class ExtractAssembly(openpype.api.Extractor): json.dump(instance.data["scenedata"], filepath, ensure_ascii=False) self.log.info("Extracting point cache ..") - cmds.select(instance.data["hierarchy"]) + cmds.select(instance.data["nodesHierarchy"]) # Run basic alembic exporter extract_alembic(file=hierarchy_path, diff --git a/openpype/hosts/maya/plugins/publish/validate_assembly_transforms.py b/openpype/hosts/maya/plugins/publish/validate_assembly_transforms.py index f793846555..fb25b617be 100644 --- a/openpype/hosts/maya/plugins/publish/validate_assembly_transforms.py +++ b/openpype/hosts/maya/plugins/publish/validate_assembly_transforms.py @@ -48,7 +48,7 @@ class ValidateAssemblyModelTransforms(pyblish.api.InstancePlugin): from openpype.hosts.maya.api import lib # Get all transforms in the loaded containers - container_roots = cmds.listRelatives(instance.data["hierarchy"], + container_roots = cmds.listRelatives(instance.data["nodesHierarchy"], children=True, type="transform", fullPath=True) From 01d3b0b0b0e2ed72b4ff509d4e0685f44086ef85 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 8 Sep 2022 13:30:27 +0200 Subject: [PATCH 092/102] add logs about missing attributes --- openpype/tools/tray/pype_tray.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/openpype/tools/tray/pype_tray.py b/openpype/tools/tray/pype_tray.py index c32a074fd1..348573a191 100644 --- a/openpype/tools/tray/pype_tray.py +++ b/openpype/tools/tray/pype_tray.py @@ -9,11 +9,11 @@ import platform from Qt import QtCore, QtGui, QtWidgets import openpype.version -from openpype.api import ( - resources, - get_system_settings +from openpype import resources, style +from openpype.lib import ( + get_openpype_execute_args, + Logger, ) -from openpype.lib import get_openpype_execute_args, Logger from openpype.lib.openpype_version import ( op_version_control_available, get_expected_version, @@ -25,8 +25,8 @@ from openpype.lib.openpype_version import ( get_openpype_version, ) from openpype.modules import TrayModulesManager -from openpype import style from openpype.settings import ( + get_system_settings, SystemSettings, ProjectSettings, DefaultsNotDefined @@ -774,6 +774,7 @@ class PypeTrayStarter(QtCore.QObject): def main(): + log = Logger.get_logger(__name__) app = QtWidgets.QApplication.instance() if not app: app = QtWidgets.QApplication([]) @@ -783,7 +784,12 @@ def main(): "AA_UseHighDpiPixmaps" ): attr = getattr(QtCore.Qt, attr_name, None) - if attr is not None: + if attr is None: + log.debug(( + "Missing QtCore.Qt attribute \"{}\"." + " UI quality may be affected." + ).format(attr_name)) + else: app.setAttribute(attr) starter = PypeTrayStarter(app) From d7a768f718944c9322ae30e9167ea64b655be5e5 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 8 Sep 2022 15:51:24 +0200 Subject: [PATCH 093/102] turn plugin off by default --- openpype/settings/defaults/project_settings/global.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/settings/defaults/project_settings/global.json b/openpype/settings/defaults/project_settings/global.json index 9258343440..99a2e16a7c 100644 --- a/openpype/settings/defaults/project_settings/global.json +++ b/openpype/settings/defaults/project_settings/global.json @@ -4,7 +4,7 @@ "follow_workfile_version": false }, "CollectAudio": { - "enabled": true, + "enabled": false, "audio_subset_name": "audioMain" }, "CollectSceneVersion": { From 49dff63f08207eea0218cf37e4824795d08e3895 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Thu, 8 Sep 2022 17:50:55 +0200 Subject: [PATCH 094/102] Fix detection of workfile instance --- openpype/modules/deadline/abstract_submit_deadline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py index f698b7688e..512ff800ee 100644 --- a/openpype/modules/deadline/abstract_submit_deadline.py +++ b/openpype/modules/deadline/abstract_submit_deadline.py @@ -519,7 +519,7 @@ class AbstractSubmitDeadline(pyblish.api.InstancePlugin): instance = self._instance workfile_instance = self._get_workfile_instance(instance.context) - if not workfile_instance: + if workfile_instance is None: return # determine published path from Anatomy. From 3943d74f3ea3b72d08f2d2e114a30b4a5e00a515 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 9 Sep 2022 11:55:05 +0200 Subject: [PATCH 095/102] flame: adding batch action hook --- openpype/hosts/flame/api/__init__.py | 4 +- openpype/hosts/flame/api/menu.py | 50 +++++++++++++++++++ .../hosts/flame/startup/openpype_in_flame.py | 13 +++++ 3 files changed, 66 insertions(+), 1 deletion(-) diff --git a/openpype/hosts/flame/api/__init__.py b/openpype/hosts/flame/api/__init__.py index 76c1c93379..7da91d41e4 100644 --- a/openpype/hosts/flame/api/__init__.py +++ b/openpype/hosts/flame/api/__init__.py @@ -51,7 +51,8 @@ from .pipeline import ( ) from .menu import ( FlameMenuProjectConnect, - FlameMenuTimeline + FlameMenuTimeline, + FlameMenuBatch ) from .plugin import ( Creator, @@ -131,6 +132,7 @@ __all__ = [ # menu "FlameMenuProjectConnect", "FlameMenuTimeline", + "FlameMenuBatch", # plugin "Creator", diff --git a/openpype/hosts/flame/api/menu.py b/openpype/hosts/flame/api/menu.py index 7f1a6a24e2..a822059930 100644 --- a/openpype/hosts/flame/api/menu.py +++ b/openpype/hosts/flame/api/menu.py @@ -201,3 +201,53 @@ class FlameMenuTimeline(_FlameMenuApp): if self.flame: self.flame.execute_shortcut('Rescan Python Hooks') self.log.info('Rescan Python Hooks') + + +class FlameMenuBatch(_FlameMenuApp): + + # flameMenuProjectconnect app takes care of the preferences dialog as well + + def __init__(self, framework): + _FlameMenuApp.__init__(self, framework) + + def __getattr__(self, name): + def method(*args, **kwargs): + project = self.dynamic_menu_data.get(name) + if project: + self.link_project(project) + return method + + def build_menu(self): + if not self.flame: + return [] + + menu = deepcopy(self.menu) + + menu['actions'].append({ + "name": "Load...", + "execute": lambda x: self.tools_helper.show_loader() + }) + menu['actions'].append({ + "name": "Manage...", + "execute": lambda x: self.tools_helper.show_scene_inventory() + }) + menu['actions'].append({ + "name": "Library...", + "execute": lambda x: self.tools_helper.show_library_loader() + }) + return menu + + def refresh(self, *args, **kwargs): + self.rescan() + + def rescan(self, *args, **kwargs): + if not self.flame: + try: + import flame + self.flame = flame + except ImportError: + self.flame = None + + if self.flame: + self.flame.execute_shortcut('Rescan Python Hooks') + self.log.info('Rescan Python Hooks') diff --git a/openpype/hosts/flame/startup/openpype_in_flame.py b/openpype/hosts/flame/startup/openpype_in_flame.py index f2ac23b19e..60f6612b7f 100644 --- a/openpype/hosts/flame/startup/openpype_in_flame.py +++ b/openpype/hosts/flame/startup/openpype_in_flame.py @@ -73,6 +73,8 @@ def load_apps(): opfapi.FlameMenuProjectConnect(opfapi.CTX.app_framework)) opfapi.CTX.flame_apps.append( opfapi.FlameMenuTimeline(opfapi.CTX.app_framework)) + opfapi.CTX.flame_apps.append( + opfapi.FlameMenuBatch(opfapi.CTX.app_framework)) opfapi.CTX.app_framework.log.info("Apps are loaded") @@ -191,3 +193,14 @@ def get_timeline_custom_ui_actions(): openpype_install() return _build_app_menu("FlameMenuTimeline") + +def get_batch_custom_ui_actions(): + """Hook to create submenu in batch + + Returns: + list: menu object + """ + # install openpype and the host + openpype_install() + + return _build_app_menu("FlameMenuBatch") \ No newline at end of file From c622eb7a59986ebe003205d66cd2ae101e1b23eb Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 9 Sep 2022 13:54:33 +0200 Subject: [PATCH 096/102] flame: add ui to project media panel --- openpype/hosts/flame/api/__init__.py | 4 ++-- openpype/hosts/flame/api/menu.py | 2 +- openpype/hosts/flame/startup/openpype_in_flame.py | 15 ++++++++++++++- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/openpype/hosts/flame/api/__init__.py b/openpype/hosts/flame/api/__init__.py index 7da91d41e4..c00ee958b6 100644 --- a/openpype/hosts/flame/api/__init__.py +++ b/openpype/hosts/flame/api/__init__.py @@ -52,7 +52,7 @@ from .pipeline import ( from .menu import ( FlameMenuProjectConnect, FlameMenuTimeline, - FlameMenuBatch + FlameMenuUniversal ) from .plugin import ( Creator, @@ -132,7 +132,7 @@ __all__ = [ # menu "FlameMenuProjectConnect", "FlameMenuTimeline", - "FlameMenuBatch", + "FlameMenuUniversal", # plugin "Creator", diff --git a/openpype/hosts/flame/api/menu.py b/openpype/hosts/flame/api/menu.py index a822059930..f72a352bba 100644 --- a/openpype/hosts/flame/api/menu.py +++ b/openpype/hosts/flame/api/menu.py @@ -203,7 +203,7 @@ class FlameMenuTimeline(_FlameMenuApp): self.log.info('Rescan Python Hooks') -class FlameMenuBatch(_FlameMenuApp): +class FlameMenuUniversal(_FlameMenuApp): # flameMenuProjectconnect app takes care of the preferences dialog as well diff --git a/openpype/hosts/flame/startup/openpype_in_flame.py b/openpype/hosts/flame/startup/openpype_in_flame.py index 60f6612b7f..9fdc30db5d 100644 --- a/openpype/hosts/flame/startup/openpype_in_flame.py +++ b/openpype/hosts/flame/startup/openpype_in_flame.py @@ -194,6 +194,7 @@ def get_timeline_custom_ui_actions(): return _build_app_menu("FlameMenuTimeline") + def get_batch_custom_ui_actions(): """Hook to create submenu in batch @@ -203,4 +204,16 @@ def get_batch_custom_ui_actions(): # install openpype and the host openpype_install() - return _build_app_menu("FlameMenuBatch") \ No newline at end of file + return _build_app_menu("FlameMenuUniversal") + + +def get_media_panel_custom_ui_actions(): + """Hook to create submenu in desktop + + Returns: + list: menu object + """ + # install openpype and the host + openpype_install() + + return _build_app_menu("FlameMenuUniversal") From 7b78e09eaec1c7e243782c13079ff70ca6c06d23 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 9 Sep 2022 13:59:15 +0200 Subject: [PATCH 097/102] fixing name of class --- openpype/hosts/flame/startup/openpype_in_flame.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/hosts/flame/startup/openpype_in_flame.py b/openpype/hosts/flame/startup/openpype_in_flame.py index 9fdc30db5d..d07aaa6b7d 100644 --- a/openpype/hosts/flame/startup/openpype_in_flame.py +++ b/openpype/hosts/flame/startup/openpype_in_flame.py @@ -74,7 +74,7 @@ def load_apps(): opfapi.CTX.flame_apps.append( opfapi.FlameMenuTimeline(opfapi.CTX.app_framework)) opfapi.CTX.flame_apps.append( - opfapi.FlameMenuBatch(opfapi.CTX.app_framework)) + opfapi.FlameMenuUniversal(opfapi.CTX.app_framework)) opfapi.CTX.app_framework.log.info("Apps are loaded") From 42f575fca94722a25ae462a5112a843799d9aad1 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 9 Sep 2022 14:23:36 +0200 Subject: [PATCH 098/102] actions are expected as list so each application in group is stored if force not open workfile is enabled --- openpype/tools/launcher/models.py | 46 +++++++++++++++++++------------ 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/openpype/tools/launcher/models.py b/openpype/tools/launcher/models.py index 6d40d21f96..6e3b531018 100644 --- a/openpype/tools/launcher/models.py +++ b/openpype/tools/launcher/models.py @@ -281,18 +281,25 @@ class ActionModel(QtGui.QStandardItemModel): if not action_item: return - action = action_item.data(ACTION_ROLE) - actual_data = self._prepare_compare_data(action) + actions = action_item.data(ACTION_ROLE) + if not isinstance(actions, list): + actions = [actions] + + action_actions_data = [ + self._prepare_compare_data(action) + for action in actions + ] stored = self.launcher_registry.get_item("force_not_open_workfile") - if is_checked: - stored.append(actual_data) - else: - final_values = [] - for config in stored: - if config != actual_data: - final_values.append(config) - stored = final_values + for actual_data in action_actions_data: + if is_checked: + stored.append(actual_data) + else: + final_values = [] + for config in stored: + if config != actual_data: + final_values.append(config) + stored = final_values self.launcher_registry.set_item("force_not_open_workfile", stored) self.launcher_registry._get_item.cache_clear() @@ -329,21 +336,24 @@ class ActionModel(QtGui.QStandardItemModel): item (QStandardItem) stored (list) of dict """ - action = item.data(ACTION_ROLE) - if not self.is_application_action(action): + + actions = item.data(ACTION_ROLE) + if not isinstance(actions, list): + actions = [actions] + + if not self.is_application_action(actions[0]): return False - actual_data = self._prepare_compare_data(action) + action_actions_data = [ + self._prepare_compare_data(action) + for action in actions + ] for config in stored: - if config == actual_data: + if config in action_actions_data: return True - return False def _prepare_compare_data(self, action): - if isinstance(action, list) and action: - action = action[0] - compare_data = {} if action and action.label: compare_data = { From 98c065cb8b3c8f75ae0479da1a3287ebbe0b22d6 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 9 Sep 2022 14:24:02 +0200 Subject: [PATCH 099/102] change "start_last_workfile" when triggered from group --- openpype/tools/launcher/widgets.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/openpype/tools/launcher/widgets.py b/openpype/tools/launcher/widgets.py index 62599664fe..774ceb659d 100644 --- a/openpype/tools/launcher/widgets.py +++ b/openpype/tools/launcher/widgets.py @@ -312,11 +312,12 @@ class ActionBar(QtWidgets.QWidget): is_group = index.data(GROUP_ROLE) is_variant_group = index.data(VARIANT_GROUP_ROLE) + force_not_open_workfile = index.data(FORCE_NOT_OPEN_WORKFILE_ROLE) if not is_group and not is_variant_group: action = index.data(ACTION_ROLE) # Change data of application action if issubclass(action, ApplicationAction): - if index.data(FORCE_NOT_OPEN_WORKFILE_ROLE): + if force_not_open_workfile: action.data["start_last_workfile"] = False else: action.data.pop("start_last_workfile", None) @@ -385,10 +386,18 @@ class ActionBar(QtWidgets.QWidget): menu.addMenu(sub_menu) result = menu.exec_(QtGui.QCursor.pos()) - if result: - action = actions_mapping[result] - self._start_animation(index) - self.action_clicked.emit(action) + if not result: + return + + action = actions_mapping[result] + if issubclass(action, ApplicationAction): + if force_not_open_workfile: + action.data["start_last_workfile"] = False + else: + action.data.pop("start_last_workfile", None) + + self._start_animation(index) + self.action_clicked.emit(action) class ActionHistory(QtWidgets.QPushButton): From 113210a781ed5affd6e92f72d075a9522a07c0f0 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 9 Sep 2022 17:21:53 +0200 Subject: [PATCH 100/102] firx access to 'Pattern' attribute --- openpype/client/entities.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/openpype/client/entities.py b/openpype/client/entities.py index a9112ac581..43afccf2f1 100644 --- a/openpype/client/entities.py +++ b/openpype/client/entities.py @@ -14,6 +14,8 @@ from bson.objectid import ObjectId from .mongo import get_project_database, get_project_connection +PatternType = type(re.compile("")) + def _prepare_fields(fields, required_fields=None): if not fields: @@ -1054,11 +1056,11 @@ def _regex_filters(filters): for key, value in filters.items(): regexes = [] a_values = [] - if isinstance(value, re.Pattern): + if isinstance(value, PatternType): regexes.append(value) elif isinstance(value, (list, tuple, set)): for item in value: - if isinstance(item, re.Pattern): + if isinstance(item, PatternType): regexes.append(item) else: a_values.append(item) @@ -1194,7 +1196,7 @@ def get_representations( as filter. Filter ignored if 'None' is passed. version_ids (Iterable[str]): Subset ids used as parent filter. Filter ignored if 'None' is passed. - context_filters (Dict[str, List[str, re.Pattern]]): Filter by + context_filters (Dict[str, List[str, PatternType]]): Filter by representation context fields. names_by_version_ids (dict[ObjectId, list[str]]): Complex filtering using version ids and list of names under the version. @@ -1240,7 +1242,7 @@ def get_archived_representations( as filter. Filter ignored if 'None' is passed. version_ids (Iterable[str]): Subset ids used as parent filter. Filter ignored if 'None' is passed. - context_filters (Dict[str, List[str, re.Pattern]]): Filter by + context_filters (Dict[str, List[str, PatternType]]): Filter by representation context fields. names_by_version_ids (dict[ObjectId, List[str]]): Complex filtering using version ids and list of names under the version. From c5ae7e5d6cb704794e444addf93bb9e903795dc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= <33513211+antirotor@users.noreply.github.com> Date: Fri, 9 Sep 2022 17:45:44 +0200 Subject: [PATCH 101/102] Update openpype/hosts/flame/plugins/publish/extract_subset_resources.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- .../hosts/flame/plugins/publish/extract_subset_resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/hosts/flame/plugins/publish/extract_subset_resources.py b/openpype/hosts/flame/plugins/publish/extract_subset_resources.py index 1af6b00654..1d42330e23 100644 --- a/openpype/hosts/flame/plugins/publish/extract_subset_resources.py +++ b/openpype/hosts/flame/plugins/publish/extract_subset_resources.py @@ -96,7 +96,7 @@ class ExtractSubsetResources(openpype.api.Extractor): source_start_handles = instance.data["sourceStartH"] source_end_handles = instance.data["sourceEndH"] - # retime if needed + # retime if needed if r_speed != 1.0: if retimed_handles: # handles are retimed From 87cb5e25b82d3ea352b17794e231924bdbb097ed Mon Sep 17 00:00:00 2001 From: OpenPype Date: Sat, 10 Sep 2022 04:17:04 +0000 Subject: [PATCH 102/102] [Automated] Bump version --- CHANGELOG.md | 45 ++++++++++++++++++--------------------------- openpype/version.py | 2 +- 2 files changed, 19 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6754f1e2e3..0ffb6a996b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,29 +1,36 @@ # Changelog -## [3.14.2-nightly.3](https://github.com/pypeclub/OpenPype/tree/HEAD) +## [3.14.2-nightly.4](https://github.com/pypeclub/OpenPype/tree/HEAD) [Full Changelog](https://github.com/pypeclub/OpenPype/compare/3.14.1...HEAD) **🆕 New features** - Nuke: Build workfile by template [\#3763](https://github.com/pypeclub/OpenPype/pull/3763) +- Houdini: Publishing workfiles [\#3697](https://github.com/pypeclub/OpenPype/pull/3697) +- Global: making collect audio plugin global [\#3679](https://github.com/pypeclub/OpenPype/pull/3679) **🚀 Enhancements** +- Flame: Adding Creator's retimed shot and handles switch [\#3826](https://github.com/pypeclub/OpenPype/pull/3826) +- Flame: OpenPype submenu to batch and media manager [\#3825](https://github.com/pypeclub/OpenPype/pull/3825) +- General: Better pixmap scaling [\#3809](https://github.com/pypeclub/OpenPype/pull/3809) - Photoshop: attempt to speed up ExtractImage [\#3793](https://github.com/pypeclub/OpenPype/pull/3793) - SyncServer: Added cli commands for sync server [\#3765](https://github.com/pypeclub/OpenPype/pull/3765) -- Blender: Publisher collect workfile representation [\#3670](https://github.com/pypeclub/OpenPype/pull/3670) -- Maya: move set render settings menu entry [\#3669](https://github.com/pypeclub/OpenPype/pull/3669) -- Scene Inventory: Maya add actions to select from or to scene [\#3659](https://github.com/pypeclub/OpenPype/pull/3659) +- Kitsu: Drop 'entities root' setting. [\#3739](https://github.com/pypeclub/OpenPype/pull/3739) **🐛 Bug fixes** +- General: Fix Pattern access in client code [\#3828](https://github.com/pypeclub/OpenPype/pull/3828) +- Launcher: Skip opening last work file works for groups [\#3822](https://github.com/pypeclub/OpenPype/pull/3822) +- Maya: Publishing data key change [\#3811](https://github.com/pypeclub/OpenPype/pull/3811) +- Igniter: Fix status handling when version is already installed [\#3804](https://github.com/pypeclub/OpenPype/pull/3804) - Resolve: Addon import is Python 2 compatible [\#3798](https://github.com/pypeclub/OpenPype/pull/3798) +- Hiero: retimed clip publishing is working [\#3792](https://github.com/pypeclub/OpenPype/pull/3792) - nuke: validate write node is not failing due wrong type [\#3780](https://github.com/pypeclub/OpenPype/pull/3780) - Fix - changed format of version string in pyproject.toml [\#3777](https://github.com/pypeclub/OpenPype/pull/3777) - Ftrack status fix typo prgoress -\> progress [\#3761](https://github.com/pypeclub/OpenPype/pull/3761) - Fix version resolution [\#3757](https://github.com/pypeclub/OpenPype/pull/3757) -- Maya: `containerise` dont skip empty values [\#3674](https://github.com/pypeclub/OpenPype/pull/3674) **🔀 Refactored code** @@ -33,17 +40,19 @@ - General: Remove unused teshost [\#3773](https://github.com/pypeclub/OpenPype/pull/3773) - General: Copied 'Extractor' plugin to publish pipeline [\#3771](https://github.com/pypeclub/OpenPype/pull/3771) - General: Move queries of asset and representation links [\#3770](https://github.com/pypeclub/OpenPype/pull/3770) +- General: Move create project folders to pipeline [\#3768](https://github.com/pypeclub/OpenPype/pull/3768) - General: Create project function moved to client code [\#3766](https://github.com/pypeclub/OpenPype/pull/3766) -- General: Move delivery logic to pipeline [\#3751](https://github.com/pypeclub/OpenPype/pull/3751) - General: Move hostdirname functionality into host [\#3749](https://github.com/pypeclub/OpenPype/pull/3749) - General: Move publish utils to pipeline [\#3745](https://github.com/pypeclub/OpenPype/pull/3745) - Houdini: Define houdini as addon [\#3735](https://github.com/pypeclub/OpenPype/pull/3735) +- Fusion: Defined fusion as addon [\#3733](https://github.com/pypeclub/OpenPype/pull/3733) - Flame: Defined flame as addon [\#3732](https://github.com/pypeclub/OpenPype/pull/3732) - Resolve: Define resolve as addon [\#3727](https://github.com/pypeclub/OpenPype/pull/3727) **Merged pull requests:** - Standalone Publisher: Ignore empty labels, then still use name like other asset models [\#3779](https://github.com/pypeclub/OpenPype/pull/3779) +- Kitsu - sync\_all\_project - add list ignore\_projects [\#3776](https://github.com/pypeclub/OpenPype/pull/3776) ## [3.14.1](https://github.com/pypeclub/OpenPype/tree/3.14.1) (2022-08-30) @@ -52,23 +61,16 @@ ### 📖 Documentation - Documentation: Few updates [\#3698](https://github.com/pypeclub/OpenPype/pull/3698) -- Documentation: Settings development [\#3660](https://github.com/pypeclub/OpenPype/pull/3660) - -**🆕 New features** - -- Webpublisher:change create flatten image into tri state [\#3678](https://github.com/pypeclub/OpenPype/pull/3678) -- Blender: validators code correction with settings and defaults [\#3662](https://github.com/pypeclub/OpenPype/pull/3662) **🚀 Enhancements** - General: Thumbnail can use project roots [\#3750](https://github.com/pypeclub/OpenPype/pull/3750) +- git: update gitignore [\#3722](https://github.com/pypeclub/OpenPype/pull/3722) - Settings: Remove settings lock on tray exit [\#3720](https://github.com/pypeclub/OpenPype/pull/3720) - General: Added helper getters to modules manager [\#3712](https://github.com/pypeclub/OpenPype/pull/3712) - Unreal: Define unreal as module and use host class [\#3701](https://github.com/pypeclub/OpenPype/pull/3701) - Settings: Lock settings UI session [\#3700](https://github.com/pypeclub/OpenPype/pull/3700) - General: Benevolent context label collector [\#3686](https://github.com/pypeclub/OpenPype/pull/3686) -- Ftrack: Store ftrack entities on hierarchy integration to instances [\#3677](https://github.com/pypeclub/OpenPype/pull/3677) -- Blender: ops refresh manager after process events [\#3663](https://github.com/pypeclub/OpenPype/pull/3663) **🐛 Bug fixes** @@ -82,11 +84,11 @@ - Settings: Fix project overrides save [\#3708](https://github.com/pypeclub/OpenPype/pull/3708) - Workfiles tool: Fix published workfile filtering [\#3704](https://github.com/pypeclub/OpenPype/pull/3704) - PS, AE: Provide default variant value for workfile subset [\#3703](https://github.com/pypeclub/OpenPype/pull/3703) -- Flame: retime is working on clip publishing [\#3684](https://github.com/pypeclub/OpenPype/pull/3684) - Webpublisher: added check for empty context [\#3682](https://github.com/pypeclub/OpenPype/pull/3682) **🔀 Refactored code** +- General: Move delivery logic to pipeline [\#3751](https://github.com/pypeclub/OpenPype/pull/3751) - General: Host addons cleanup [\#3744](https://github.com/pypeclub/OpenPype/pull/3744) - Webpublisher: Webpublisher is used as addon [\#3740](https://github.com/pypeclub/OpenPype/pull/3740) - Photoshop: Defined photoshop as addon [\#3736](https://github.com/pypeclub/OpenPype/pull/3736) @@ -110,7 +112,6 @@ - Hiero: Define hiero as module [\#3717](https://github.com/pypeclub/OpenPype/pull/3717) - Deadline: better logging for DL webservice failures [\#3694](https://github.com/pypeclub/OpenPype/pull/3694) -- Photoshop: resize saved images in ExtractReview for ffmpeg [\#3676](https://github.com/pypeclub/OpenPype/pull/3676) ## [3.14.0](https://github.com/pypeclub/OpenPype/tree/3.14.0) (2022-08-18) @@ -120,21 +121,11 @@ - Ftrack: Addiotional component metadata [\#3685](https://github.com/pypeclub/OpenPype/pull/3685) - Ftrack: Set task status on farm publishing [\#3680](https://github.com/pypeclub/OpenPype/pull/3680) -- Ftrack: Set task status on task creation in integrate hierarchy [\#3675](https://github.com/pypeclub/OpenPype/pull/3675) -- Maya: Disable rendering of all lights for render instances submitted through Deadline. [\#3661](https://github.com/pypeclub/OpenPype/pull/3661) **🐛 Bug fixes** - General: Switch from hero version to versioned works [\#3691](https://github.com/pypeclub/OpenPype/pull/3691) -- General: Fix finding of last version [\#3656](https://github.com/pypeclub/OpenPype/pull/3656) - -**🔀 Refactored code** - -- General: Use client projects getter [\#3673](https://github.com/pypeclub/OpenPype/pull/3673) - -**Merged pull requests:** - -- Deadline: Global job pre load is not Pype 2 compatible [\#3666](https://github.com/pypeclub/OpenPype/pull/3666) +- Flame: retime is working on clip publishing [\#3684](https://github.com/pypeclub/OpenPype/pull/3684) ## [3.13.0](https://github.com/pypeclub/OpenPype/tree/3.13.0) (2022-08-09) diff --git a/openpype/version.py b/openpype/version.py index c042ca2625..142bd51a30 100644 --- a/openpype/version.py +++ b/openpype/version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- """Package declaring Pype version.""" -__version__ = "3.14.2-nightly.3" +__version__ = "3.14.2-nightly.4"