From 551f34a873c89e739dc0b5d28a74eeec3f79dac2 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Fri, 12 Aug 2022 14:25:29 +0200 Subject: [PATCH 001/122] Add subsetGroup column to scene inventory --- openpype/tools/sceneinventory/model.py | 11 +++++++++-- openpype/tools/sceneinventory/window.py | 3 ++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/openpype/tools/sceneinventory/model.py b/openpype/tools/sceneinventory/model.py index 63fbe04c5c..97cc11ff23 100644 --- a/openpype/tools/sceneinventory/model.py +++ b/openpype/tools/sceneinventory/model.py @@ -34,7 +34,8 @@ from .lib import ( class InventoryModel(TreeModel): """The model for the inventory""" - Columns = ["Name", "version", "count", "family", "loader", "objectName"] + Columns = ["Name", "version", "count", "family", + "subsetGroup", "loader", "objectName"] OUTDATED_COLOR = QtGui.QColor(235, 30, 30) CHILD_OUTDATED_COLOR = QtGui.QColor(200, 160, 30) @@ -157,8 +158,13 @@ class InventoryModel(TreeModel): # Family icon return item.get("familyIcon", None) + column_name = self.Columns[index.column()] + + if column_name == "subsetGroup" and item.get("subsetGroup"): + return qtawesome.icon("fa.object-group", + color=get_default_entity_icon_color()) + if item.get("isGroupNode"): - column_name = self.Columns[index.column()] if column_name == "active_site": provider = item.get("active_site_provider") return self._site_icons.get(provider) @@ -423,6 +429,7 @@ class InventoryModel(TreeModel): group_node["familyIcon"] = family_icon group_node["count"] = len(group_items) group_node["isGroupNode"] = True + group_node["subsetGroup"] = subset["data"].get("subsetGroup") if self.sync_enabled: progress = get_progress_for_repre( diff --git a/openpype/tools/sceneinventory/window.py b/openpype/tools/sceneinventory/window.py index 054c2a2daa..02addbccfe 100644 --- a/openpype/tools/sceneinventory/window.py +++ b/openpype/tools/sceneinventory/window.py @@ -88,7 +88,8 @@ class SceneInventoryWindow(QtWidgets.QDialog): view.setColumnWidth(1, 55) # version view.setColumnWidth(2, 55) # count view.setColumnWidth(3, 150) # family - view.setColumnWidth(4, 100) # namespace + view.setColumnWidth(4, 120) # subsetGroup + view.setColumnWidth(5, 150) # loader # apply delegates version_delegate = VersionDelegate(legacy_io, self) From 2cebaf718cc3c7df21a3543e90bac91ae56e68d8 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 15 Aug 2022 21:04:51 +0200 Subject: [PATCH 002/122] global: moving collect audio to global --- .../plugins/publish/collect_audio.py | 113 ------------------ openpype/plugins/publish/collect_audio.py | 51 ++++++++ 2 files changed, 51 insertions(+), 113 deletions(-) delete mode 100644 openpype/hosts/celaction/plugins/publish/collect_audio.py create mode 100644 openpype/plugins/publish/collect_audio.py diff --git a/openpype/hosts/celaction/plugins/publish/collect_audio.py b/openpype/hosts/celaction/plugins/publish/collect_audio.py deleted file mode 100644 index c6e3bf2c03..0000000000 --- a/openpype/hosts/celaction/plugins/publish/collect_audio.py +++ /dev/null @@ -1,113 +0,0 @@ -import os -import collections -from pprint import pformat - -import pyblish.api - -from openpype.client import ( - get_subsets, - get_last_versions, - get_representations -) -from openpype.pipeline import legacy_io - - -class AppendCelactionAudio(pyblish.api.ContextPlugin): - - label = "Colect Audio for publishing" - order = pyblish.api.CollectorOrder + 0.1 - - def process(self, context): - self.log.info('Collecting Audio Data') - asset_doc = context.data["assetEntity"] - - # get all available representations - subsets = self.get_subsets( - asset_doc, - representations=["audio", "wav"] - ) - self.log.info(f"subsets is: {pformat(subsets)}") - - if not subsets.get("audioMain"): - raise AttributeError("`audioMain` subset does not exist") - - reprs = subsets.get("audioMain", {}).get("representations", []) - self.log.info(f"reprs is: {pformat(reprs)}") - - repr = next((r for r in reprs), None) - if not repr: - raise "Missing `audioMain` representation" - self.log.info(f"representation is: {repr}") - - audio_file = repr.get('data', {}).get('path', "") - - if os.path.exists(audio_file): - context.data["audioFile"] = audio_file - self.log.info( - 'audio_file: {}, has been added to context'.format(audio_file)) - else: - self.log.warning("Couldn't find any audio file on Ftrack.") - - def get_subsets(self, asset_doc, representations): - """ - Query subsets with filter on name. - - The method will return all found subsets and its defined version - and subsets. Version could be specified with number. Representation - can be filtered. - - Arguments: - asset_doct (dict): Asset (shot) mongo document - representations (list): list for all representations - - Returns: - dict: subsets with version and representations in keys - """ - - # Query all subsets for asset - project_name = legacy_io.active_project() - subset_docs = get_subsets( - project_name, asset_ids=[asset_doc["_id"]], fields=["_id"] - ) - # Collect all subset ids - subset_ids = [ - subset_doc["_id"] - for subset_doc in subset_docs - ] - - # Check if we found anything - assert subset_ids, ( - "No subsets found. Check correct filter. " - "Try this for start `r'.*'`: asset: `{}`" - ).format(asset_doc["name"]) - - last_versions_by_subset_id = get_last_versions( - project_name, subset_ids, fields=["_id", "parent"] - ) - - version_docs_by_id = {} - for version_doc in last_versions_by_subset_id.values(): - version_docs_by_id[version_doc["_id"]] = version_doc - - repre_docs = get_representations( - project_name, - version_ids=version_docs_by_id.keys(), - representation_names=representations - ) - repre_docs_by_version_id = collections.defaultdict(list) - for repre_doc in repre_docs: - version_id = repre_doc["parent"] - repre_docs_by_version_id[version_id].append(repre_doc) - - output_dict = {} - for version_id, repre_docs in repre_docs_by_version_id.items(): - version_doc = version_docs_by_id[version_id] - subset_id = version_doc["parent"] - subset_doc = last_versions_by_subset_id[subset_id] - # Store queried docs by subset name - output_dict[subset_doc["name"]] = { - "representations": repre_docs, - "version": version_doc - } - - return output_dict diff --git a/openpype/plugins/publish/collect_audio.py b/openpype/plugins/publish/collect_audio.py new file mode 100644 index 0000000000..022334e0f3 --- /dev/null +++ b/openpype/plugins/publish/collect_audio.py @@ -0,0 +1,51 @@ +import pyblish.api +from pprint import pformat + +from openpype.client import ( + get_last_version_by_subset_name, + get_representations, +) +from openpype.pipeline import ( + legacy_io, + get_representation_path, +) + + +class CollectAudio(pyblish.api.InstancePlugin): + + label = "Colect Audio" + order = pyblish.api.CollectorOrder + 0.1 + hosts = ["standalonepublisher"] + + def process(self, instance): + self.log.info('Collecting Audio Data') + + project_name = legacy_io.active_project() + asset_name = instance.data["asset"] + # * Add audio to instance if exists. + # Find latest versions document + last_version_doc = get_last_version_by_subset_name( + project_name, "audioMain", asset_name=asset_name, fields=["_id"] + ) + + repre_doc = None + if last_version_doc: + # Try to find it's representation (Expected there is only one) + repre_docs = list(get_representations( + project_name, version_ids=[last_version_doc["_id"]] + )) + if not repre_docs: + self.log.warning( + "Version document does not contain any representations" + ) + else: + repre_doc = repre_docs[0] + + # Add audio to instance if representation was found + if repre_doc: + instance.data["audio"] = [{ + "offset": 0, + "filename": get_representation_path(repre_doc) + }] + + self.log.debug("instance.data: {}".format(pformat(instance.data))) From 7c06a1fe8cbbdd612fb9a6a8f6b8df092f8e810a Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 15 Aug 2022 21:06:40 +0200 Subject: [PATCH 003/122] global: improving hosts and families in collect audio --- openpype/plugins/publish/collect_audio.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/openpype/plugins/publish/collect_audio.py b/openpype/plugins/publish/collect_audio.py index 022334e0f3..f4cad86f94 100644 --- a/openpype/plugins/publish/collect_audio.py +++ b/openpype/plugins/publish/collect_audio.py @@ -15,7 +15,24 @@ class CollectAudio(pyblish.api.InstancePlugin): label = "Colect Audio" order = pyblish.api.CollectorOrder + 0.1 - hosts = ["standalonepublisher"] + families = ["review"] + hosts = [ + "nuke", + "maya", + "shell", + "hiero", + "premiere", + "harmony", + "traypublisher", + "standalonepublisher", + "fusion", + "tvpaint", + "resolve", + "webpublisher", + "aftereffects", + "flame", + "unreal" + ] def process(self, instance): self.log.info('Collecting Audio Data') From 495a65707cfa7384dcd1498863a34942cb85a5ac Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 15 Aug 2022 21:13:09 +0200 Subject: [PATCH 004/122] Global: improving docstring and comments --- openpype/plugins/publish/collect_audio.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/openpype/plugins/publish/collect_audio.py b/openpype/plugins/publish/collect_audio.py index f4cad86f94..4ba47f739d 100644 --- a/openpype/plugins/publish/collect_audio.py +++ b/openpype/plugins/publish/collect_audio.py @@ -12,7 +12,9 @@ from openpype.pipeline import ( class CollectAudio(pyblish.api.InstancePlugin): + """ Collecting available audio subset to instance + """ label = "Colect Audio" order = pyblish.api.CollectorOrder + 0.1 families = ["review"] @@ -35,11 +37,12 @@ class CollectAudio(pyblish.api.InstancePlugin): ] def process(self, instance): + # * Add audio to instance if exists. self.log.info('Collecting Audio Data') project_name = legacy_io.active_project() asset_name = instance.data["asset"] - # * Add audio to instance if exists. + # Find latest versions document last_version_doc = get_last_version_by_subset_name( project_name, "audioMain", asset_name=asset_name, fields=["_id"] From 74934a51b9ce7d581473426ef9206fad1ab4b486 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 16 Aug 2022 17:21:37 +0200 Subject: [PATCH 005/122] Nuke: removing audio inclusion from precollect write --- .../nuke/plugins/publish/precollect_writes.py | 28 ------------------- 1 file changed, 28 deletions(-) diff --git a/openpype/hosts/nuke/plugins/publish/precollect_writes.py b/openpype/hosts/nuke/plugins/publish/precollect_writes.py index e37cc8a80a..17c4bc30cf 100644 --- a/openpype/hosts/nuke/plugins/publish/precollect_writes.py +++ b/openpype/hosts/nuke/plugins/publish/precollect_writes.py @@ -201,34 +201,6 @@ class CollectNukeWrites(pyblish.api.InstancePlugin): if not instance.data["review"]: instance.data["useSequenceForReview"] = False - project_name = legacy_io.active_project() - asset_name = instance.data["asset"] - # * Add audio to instance if exists. - # Find latest versions document - last_version_doc = get_last_version_by_subset_name( - project_name, "audioMain", asset_name=asset_name, fields=["_id"] - ) - - repre_doc = None - if last_version_doc: - # Try to find it's representation (Expected there is only one) - repre_docs = list(get_representations( - project_name, version_ids=[last_version_doc["_id"]] - )) - if not repre_docs: - self.log.warning( - "Version document does not contain any representations" - ) - else: - repre_doc = repre_docs[0] - - # Add audio to instance if representation was found - if repre_doc: - instance.data["audio"] = [{ - "offset": 0, - "filename": get_representation_path(repre_doc) - }] - self.log.debug("instance.data: {}".format(pformat(instance.data))) def is_prerender(self, families): From 737edadfd5166eb20197d818eefe2a1a22041b0d Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 16 Aug 2022 17:23:42 +0200 Subject: [PATCH 006/122] global: preparation for settings attribute --- openpype/plugins/publish/collect_audio.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/openpype/plugins/publish/collect_audio.py b/openpype/plugins/publish/collect_audio.py index 4ba47f739d..7e3b42f375 100644 --- a/openpype/plugins/publish/collect_audio.py +++ b/openpype/plugins/publish/collect_audio.py @@ -36,6 +36,8 @@ class CollectAudio(pyblish.api.InstancePlugin): "unreal" ] + audio_subset_name = "audioMain" + def process(self, instance): # * Add audio to instance if exists. self.log.info('Collecting Audio Data') @@ -45,7 +47,10 @@ class CollectAudio(pyblish.api.InstancePlugin): # Find latest versions document last_version_doc = get_last_version_by_subset_name( - project_name, "audioMain", asset_name=asset_name, fields=["_id"] + project_name, + self.audio_subset_name, + asset_name=asset_name, + fields=["_id"] ) repre_doc = None From 266975d6942156e71565a3847e4e62a9490dee71 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 16 Aug 2022 17:36:39 +0200 Subject: [PATCH 007/122] settings: adding collect audio plugin --- .../defaults/project_settings/global.json | 4 ++++ .../schemas/schema_global_publish.json | 21 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/openpype/settings/defaults/project_settings/global.json b/openpype/settings/defaults/project_settings/global.json index 0ff9363ba7..9258343440 100644 --- a/openpype/settings/defaults/project_settings/global.json +++ b/openpype/settings/defaults/project_settings/global.json @@ -3,6 +3,10 @@ "CollectAnatomyInstanceData": { "follow_workfile_version": false }, + "CollectAudio": { + "enabled": true, + "audio_subset_name": "audioMain" + }, "CollectSceneVersion": { "hosts": [ "aftereffects", diff --git a/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_publish.json b/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_publish.json index e1aa230b49..2efee92832 100644 --- a/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_publish.json +++ b/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_publish.json @@ -18,6 +18,27 @@ } ] }, + { + "type": "dict", + "collapsible": true, + "checkbox_key": "enabled", + "key": "CollectAudio", + "label": "Collect Audio", + "is_group": true, + "children": [ + { + "type": "boolean", + "key": "enabled", + "label": "Enabled" + }, + { + "key": "audio_subset_name", + "label": "Name of audio variant", + "type": "text", + "placeholder": "audioMain" + } + ] + }, { "type": "dict", "collapsible": true, From 98ac7f538e6482007115aff917cdf4ccd39fbc83 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 17 Aug 2022 11:59:15 +0200 Subject: [PATCH 008/122] condition for case where audio is already collected --- openpype/plugins/publish/collect_audio.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/openpype/plugins/publish/collect_audio.py b/openpype/plugins/publish/collect_audio.py index 7e3b42f375..6aed3f82fe 100644 --- a/openpype/plugins/publish/collect_audio.py +++ b/openpype/plugins/publish/collect_audio.py @@ -68,9 +68,10 @@ class CollectAudio(pyblish.api.InstancePlugin): # Add audio to instance if representation was found if repre_doc: - instance.data["audio"] = [{ - "offset": 0, - "filename": get_representation_path(repre_doc) - }] + if not instance.data.get("audio"): + instance.data["audio"] = [{ + "offset": 0, + "filename": get_representation_path(repre_doc) + }] self.log.debug("instance.data: {}".format(pformat(instance.data))) From ed2aedd0feec05d7d53f2e62789f77838a6f2f47 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 18 Aug 2022 10:17:02 +0200 Subject: [PATCH 009/122] processing PR comments --- openpype/plugins/publish/collect_audio.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/openpype/plugins/publish/collect_audio.py b/openpype/plugins/publish/collect_audio.py index 6aed3f82fe..cf074392ee 100644 --- a/openpype/plugins/publish/collect_audio.py +++ b/openpype/plugins/publish/collect_audio.py @@ -15,7 +15,7 @@ class CollectAudio(pyblish.api.InstancePlugin): """ Collecting available audio subset to instance """ - label = "Colect Audio" + label = "Collect Audio" order = pyblish.api.CollectorOrder + 0.1 families = ["review"] hosts = [ @@ -39,8 +39,14 @@ class CollectAudio(pyblish.api.InstancePlugin): audio_subset_name = "audioMain" def process(self, instance): - # * Add audio to instance if exists. - self.log.info('Collecting Audio Data') + if instance.data.get("audio"): + self.log.info( + "Skipping Audio collecion. It is already collected" + ) + return + + # Add audio to instance if exists. + self.log.info('Collecting Audio Data ...') project_name = legacy_io.active_project() asset_name = instance.data["asset"] @@ -68,10 +74,10 @@ class CollectAudio(pyblish.api.InstancePlugin): # Add audio to instance if representation was found if repre_doc: - if not instance.data.get("audio"): - instance.data["audio"] = [{ - "offset": 0, - "filename": get_representation_path(repre_doc) - }] + instance.data["audio"] = [{ + "offset": 0, + "filename": get_representation_path(repre_doc) + }] + self.log.info("Audio Data added to instance ...") self.log.debug("instance.data: {}".format(pformat(instance.data))) From 694a07579287b12b5fcce35fc4a405700ca3f64c Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Thu, 18 Aug 2022 11:45:01 +0200 Subject: [PATCH 010/122] Refactor `subsetGroup` column name to `group` Co-authored-by: Milan Kolar --- openpype/tools/sceneinventory/model.py | 6 +++--- openpype/tools/sceneinventory/window.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/openpype/tools/sceneinventory/model.py b/openpype/tools/sceneinventory/model.py index 97cc11ff23..1a3b7c7055 100644 --- a/openpype/tools/sceneinventory/model.py +++ b/openpype/tools/sceneinventory/model.py @@ -35,7 +35,7 @@ class InventoryModel(TreeModel): """The model for the inventory""" Columns = ["Name", "version", "count", "family", - "subsetGroup", "loader", "objectName"] + "group", "loader", "objectName"] OUTDATED_COLOR = QtGui.QColor(235, 30, 30) CHILD_OUTDATED_COLOR = QtGui.QColor(200, 160, 30) @@ -160,7 +160,7 @@ class InventoryModel(TreeModel): column_name = self.Columns[index.column()] - if column_name == "subsetGroup" and item.get("subsetGroup"): + if column_name == "group" and item.get("group"): return qtawesome.icon("fa.object-group", color=get_default_entity_icon_color()) @@ -429,7 +429,7 @@ class InventoryModel(TreeModel): group_node["familyIcon"] = family_icon group_node["count"] = len(group_items) group_node["isGroupNode"] = True - group_node["subsetGroup"] = subset["data"].get("subsetGroup") + group_node["group"] = subset["data"].get("subsetGroup") if self.sync_enabled: progress = get_progress_for_repre( diff --git a/openpype/tools/sceneinventory/window.py b/openpype/tools/sceneinventory/window.py index 02addbccfe..1f4585b650 100644 --- a/openpype/tools/sceneinventory/window.py +++ b/openpype/tools/sceneinventory/window.py @@ -88,7 +88,7 @@ class SceneInventoryWindow(QtWidgets.QDialog): view.setColumnWidth(1, 55) # version view.setColumnWidth(2, 55) # count view.setColumnWidth(3, 150) # family - view.setColumnWidth(4, 120) # subsetGroup + view.setColumnWidth(4, 120) # group view.setColumnWidth(5, 150) # loader # apply delegates From 149b65b8b05750966ab5d93a5751e7085cf81652 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 23 Aug 2022 16:14:38 +0200 Subject: [PATCH 011/122] global: audio PR comments --- openpype/plugins/publish/collect_audio.py | 45 +++++++++++++++++------ 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/openpype/plugins/publish/collect_audio.py b/openpype/plugins/publish/collect_audio.py index cf074392ee..3a765d345d 100644 --- a/openpype/plugins/publish/collect_audio.py +++ b/openpype/plugins/publish/collect_audio.py @@ -12,10 +12,12 @@ from openpype.pipeline import ( class CollectAudio(pyblish.api.InstancePlugin): - """ Collecting available audio subset to instance + """Collect asset's last published audio. + The audio subset name searched for is defined in: + project settings > Collect Audio """ - label = "Collect Audio" + label = "Collect Asset Audio" order = pyblish.api.CollectorOrder + 0.1 families = ["review"] hosts = [ @@ -46,10 +48,33 @@ class CollectAudio(pyblish.api.InstancePlugin): return # Add audio to instance if exists. - self.log.info('Collecting Audio Data ...') + self.log.info(( + "Searching for audio subset '{subset}'" + " in asset '{asset}'" + ).format( + subset=self.audio_subset_name, + asset=instance.data["asset"] + )) + + repre_doc = self._get_repre_doc(instance) + + # Add audio to instance if representation was found + if repre_doc: + instance.data["audio"] = [{ + "offset": 0, + "filename": get_representation_path(repre_doc) + }] + self.log.info("Audio Data added to instance ...") + + def _get_repre_doc(self, instance): + cache = instance.context.data.get("__cache_asset_audio", {}) + asset_name = instance.data["asset"] + + # first try to get it from cache + if asset_name in cache: + return cache[asset_name] project_name = legacy_io.active_project() - asset_name = instance.data["asset"] # Find latest versions document last_version_doc = get_last_version_by_subset_name( @@ -72,12 +97,8 @@ class CollectAudio(pyblish.api.InstancePlugin): else: repre_doc = repre_docs[0] - # Add audio to instance if representation was found - if repre_doc: - instance.data["audio"] = [{ - "offset": 0, - "filename": get_representation_path(repre_doc) - }] - self.log.info("Audio Data added to instance ...") + # update cache + cache[asset_name] = repre_doc + instance.context.data["__cache_asset_audio"].update(cache) - self.log.debug("instance.data: {}".format(pformat(instance.data))) + return repre_doc From d2d90ed2e098587cd466243f5d666093ed1db55f Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 23 Aug 2022 17:30:43 +0200 Subject: [PATCH 012/122] hound catch --- openpype/plugins/publish/collect_audio.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openpype/plugins/publish/collect_audio.py b/openpype/plugins/publish/collect_audio.py index 3a765d345d..e2fb766ec4 100644 --- a/openpype/plugins/publish/collect_audio.py +++ b/openpype/plugins/publish/collect_audio.py @@ -1,5 +1,4 @@ import pyblish.api -from pprint import pformat from openpype.client import ( get_last_version_by_subset_name, From 65d785d100c986128a88f1dc2c77b5321b85d7da Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Wed, 24 Aug 2022 00:29:08 +0200 Subject: [PATCH 013/122] Draft stash for refactoring maya submit deadline to use `AbstractSubmitDeadline` base. - This does *NOT* work currently! --- .../deadline/abstract_submit_deadline.py | 22 + .../plugins/publish/submit_maya_deadline.py | 1297 ++++++++--------- 2 files changed, 623 insertions(+), 696 deletions(-) diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py index 0bad981fdf..577378335e 100644 --- a/openpype/modules/deadline/abstract_submit_deadline.py +++ b/openpype/modules/deadline/abstract_submit_deadline.py @@ -359,6 +359,27 @@ class DeadlineJobInfo(object): def OutputDirectory(self, val): # noqa: N802 self._outputDirectory.append(val) + # Asset Dependency + # ---------------------------------------------- + _assetDependency = attr.ib(factory=list) + + @property + def AssetDependency(self): # noqa: N802 + """Return all OutputDirectory values formatted for Deadline. + + Returns: + dict: as `{'OutputDirectory0': 'dir'}` + + """ + out = {} + for index, v in enumerate(self._assetDependency): + out["AssetDependency{}".format(index)] = v + return out + + @OutputDirectory.setter + def AssetDependency(self, val): # noqa: N802 + self._assetDependency.append(val) + # Tile Job # ---------------------------------------------- TileJob = attr.ib(default=None) # Default: false @@ -396,6 +417,7 @@ class DeadlineJobInfo(object): serialized.update(self.OutputFilename) serialized.update(self.OutputFilenameTile) serialized.update(self.OutputDirectory) + serialized.update(self.AssetDependency) return serialized diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 7966861358..6dfa48a9f8 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -18,7 +18,6 @@ Attributes: from __future__ import print_function import os -import json import getpass import copy import re @@ -27,252 +26,32 @@ from datetime import datetime import itertools from collections import OrderedDict +import attr import clique -import requests from maya import cmds -import pyblish.api - -from openpype.lib import requests_post from openpype.hosts.maya.api import lib from openpype.pipeline import legacy_io -# Documentation for keys available at: -# https://docs.thinkboxsoftware.com -# /products/deadline/8.0/1_User%20Manual/manual -# /manual-submission.html#job-info-file-options - -payload_skeleton_template = { - "JobInfo": { - "BatchName": None, # Top-level group name - "Name": None, # Job name, as seen in Monitor - "UserName": None, - "Plugin": "MayaBatch", - "Frames": "{start}-{end}x{step}", - "Comment": None, - "Priority": 50, - }, - "PluginInfo": { - "SceneFile": None, # Input - "OutputFilePath": None, # Output directory and filename - "OutputFilePrefix": None, - "Version": cmds.about(version=True), # Mandatory for Deadline - "UsingRenderLayers": True, - "RenderLayer": None, # Render only this layer - "Renderer": None, - "ProjectPath": None, # Resolve relative references - "RenderSetupIncludeLights": None, # Include all lights flag. - }, - "AuxFiles": [] # Mandatory for Deadline, may be empty -} +from openpype_modules.deadline import abstract_submit_deadline +from openpype_modules.deadline.abstract_submit_deadline import DeadlineJobInfo -def _format_tiles( - filename, index, tiles_x, tiles_y, - width, height, prefix): - """Generate tile entries for Deadline tile job. - - Returns two dictionaries - one that can be directly used in Deadline - job, second that can be used for Deadline Assembly job configuration - file. - - This will format tile names: - - Example:: - { - "OutputFilename0Tile0": "_tile_1x1_4x4_Main_beauty.1001.exr", - "OutputFilename0Tile1": "_tile_2x1_4x4_Main_beauty.1001.exr" - } - - And add tile prefixes like: - - Example:: - Image prefix is: - `maya///_` - - Result for tile 0 for 4x4 will be: - `maya///_tile_1x1_4x4__` - - Calculating coordinates is tricky as in Job they are defined as top, - left, bottom, right with zero being in top-left corner. But Assembler - configuration file takes tile coordinates as X, Y, Width and Height and - zero is bottom left corner. - - Args: - filename (str): Filename to process as tiles. - index (int): Index of that file if it is sequence. - tiles_x (int): Number of tiles in X. - tiles_y (int): Number if tikes in Y. - width (int): Width resolution of final image. - height (int): Height resolution of final image. - prefix (str): Image prefix. - - Returns: - (dict, dict): Tuple of two dictionaires - first can be used to - extend JobInfo, second has tiles x, y, width and height - used for assembler configuration. - - """ - tile = 0 - out = {"JobInfo": {}, "PluginInfo": {}} - cfg = OrderedDict() - w_space = width / tiles_x - h_space = height / tiles_y - - cfg["TilesCropped"] = "False" - - for tile_x in range(1, tiles_x + 1): - for tile_y in reversed(range(1, tiles_y + 1)): - tile_prefix = "_tile_{}x{}_{}x{}_".format( - tile_x, tile_y, - tiles_x, - tiles_y - ) - out_tile_index = "OutputFilename{}Tile{}".format( - str(index), tile - ) - new_filename = "{}/{}{}".format( - os.path.dirname(filename), - tile_prefix, - os.path.basename(filename) - ) - out["JobInfo"][out_tile_index] = new_filename - out["PluginInfo"]["RegionPrefix{}".format(str(tile))] = \ - "/{}".format(tile_prefix).join(prefix.rsplit("/", 1)) - - out["PluginInfo"]["RegionTop{}".format(tile)] = int(height) - (tile_y * h_space) # noqa: E501 - out["PluginInfo"]["RegionBottom{}".format(tile)] = int(height) - ((tile_y - 1) * h_space) - 1 # noqa: E501 - out["PluginInfo"]["RegionLeft{}".format(tile)] = (tile_x - 1) * w_space # noqa: E501 - out["PluginInfo"]["RegionRight{}".format(tile)] = (tile_x * w_space) - 1 # noqa: E501 - - cfg["Tile{}".format(tile)] = new_filename - cfg["Tile{}Tile".format(tile)] = new_filename - cfg["Tile{}FileName".format(tile)] = new_filename - cfg["Tile{}X".format(tile)] = (tile_x - 1) * w_space - - cfg["Tile{}Y".format(tile)] = int(height) - (tile_y * h_space) - - cfg["Tile{}Width".format(tile)] = w_space - cfg["Tile{}Height".format(tile)] = h_space - - tile += 1 - return out, cfg +@attr.s +class DeadlinePluginInfo(): + SceneFile = attr.ib(default=None) # Input + OutputFilePath = attr.ib(default=None) # Output directory and filename + OutputFilePrefix = attr.ib(default=None) + Version = attr.ib(default=None) # Mandatory for Deadline + UsingRenderLayers = attr.ib(default=True) + RenderLayer = attr.ib(default=None) # Render only this layer + Renderer = attr.ib(default=None) + ProjectPath = attr.ib(default=None) # Resolve relative references + RenderSetupIncludeLights = attr.ib(default=None) # Include all lights flag -def get_renderer_variables(renderlayer, root): - """Retrieve the extension which has been set in the VRay settings. - - Will return None if the current renderer is not VRay - For Maya 2016.5 and up the renderSetup creates renderSetupLayer node which - start with `rs`. Use the actual node name, do NOT use the `nice name` - - Args: - renderlayer (str): the node name of the renderlayer. - root (str): base path to render - - Returns: - dict - - """ - renderer = lib.get_renderer(renderlayer or lib.get_current_renderlayer()) - render_attrs = lib.RENDER_ATTRS.get(renderer, lib.RENDER_ATTRS["default"]) - - padding = cmds.getAttr("{}.{}".format(render_attrs["node"], - render_attrs["padding"])) - - filename_0 = cmds.renderSettings( - fullPath=True, - gin="#" * int(padding), - lut=True, - layer=renderlayer or lib.get_current_renderlayer())[0] - filename_0 = re.sub('_', '_beauty', - filename_0, flags=re.IGNORECASE) - prefix_attr = "defaultRenderGlobals.imageFilePrefix" - - scene = cmds.file(query=True, sceneName=True) - scene, _ = os.path.splitext(os.path.basename(scene)) - - if renderer == "vray": - renderlayer = renderlayer.split("_")[-1] - # Maya's renderSettings function does not return V-Ray file extension - # so we get the extension from vraySettings - extension = cmds.getAttr("vraySettings.imageFormatStr") - - # When V-Ray image format has not been switched once from default .png - # the getAttr command above returns None. As such we explicitly set - # it to `.png` - if extension is None: - extension = "png" - - if extension in ["exr (multichannel)", "exr (deep)"]: - extension = "exr" - - prefix_attr = "vraySettings.fileNamePrefix" - filename_prefix = cmds.getAttr(prefix_attr) - # we need to determine path for vray as maya `renderSettings` query - # does not work for vray. - - filename_0 = re.sub('', scene, filename_prefix, flags=re.IGNORECASE) # noqa: E501 - filename_0 = re.sub('', renderlayer, filename_0, flags=re.IGNORECASE) # noqa: E501 - filename_0 = "{}.{}.{}".format( - filename_0, "#" * int(padding), extension) - filename_0 = os.path.normpath(os.path.join(root, filename_0)) - elif renderer == "renderman": - prefix_attr = "rmanGlobals.imageFileFormat" - # NOTE: This is guessing extensions from renderman display types. - # Some of them are just framebuffers, d_texture format can be - # set in display setting. We set those now to None, but it - # should be handled more gracefully. - display_types = { - "d_deepexr": "exr", - "d_it": None, - "d_null": None, - "d_openexr": "exr", - "d_png": "png", - "d_pointcloud": "ptc", - "d_targa": "tga", - "d_texture": None, - "d_tiff": "tif" - } - - extension = display_types.get( - cmds.listConnections("rmanDefaultDisplay.displayType")[0], - "exr" - ) or "exr" - - filename_prefix = "{}/{}".format( - cmds.getAttr("rmanGlobals.imageOutputDir"), - cmds.getAttr("rmanGlobals.imageFileFormat") - ) - - renderlayer = renderlayer.split("_")[-1] - - filename_0 = re.sub('', scene, filename_prefix, flags=re.IGNORECASE) # noqa: E501 - filename_0 = re.sub('', renderlayer, filename_0, flags=re.IGNORECASE) # noqa: E501 - filename_0 = re.sub('', "#" * int(padding), filename_0, flags=re.IGNORECASE) # noqa: E501 - filename_0 = re.sub('', extension, filename_0, flags=re.IGNORECASE) # noqa: E501 - filename_0 = os.path.normpath(os.path.join(root, filename_0)) - elif renderer == "redshift": - # mapping redshift extension dropdown values to strings - ext_mapping = ["iff", "exr", "tif", "png", "tga", "jpg"] - extension = ext_mapping[ - cmds.getAttr("redshiftOptions.imageFormat") - ] - else: - # Get the extension, getAttr defaultRenderGlobals.imageFormat - # returns an index number. - filename_base = os.path.basename(filename_0) - extension = os.path.splitext(filename_base)[-1].strip(".") - - filename_prefix = cmds.getAttr(prefix_attr) - return {"ext": extension, - "filename_prefix": filename_prefix, - "padding": padding, - "filename_0": filename_0} - - -class MayaSubmitDeadline(pyblish.api.InstancePlugin): +class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): """Submit available render layers to Deadline. Renders are submitted to a Deadline Web Service as @@ -284,15 +63,12 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin): """ - label = "Submit to Deadline" - order = pyblish.api.IntegratorOrder + 0.1 + label = "Submit Render to Deadline" hosts = ["maya"] families = ["renderlayer"] targets = ["local"] - use_published = True tile_assembler_plugin = "OpenPypeTileAssembler" - asset_dependencies = False priority = 50 tile_priority = 50 limit_groups = [] @@ -300,32 +76,173 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin): pluginInfo = {} group = "none" - def process(self, instance): - """Plugin entry point.""" - instance.data["toBeRenderedOn"] = "deadline" + def get_job_info(self): + job_info = DeadlineJobInfo(Plugin="MayaBatch") + + # todo: test whether this works for existing production cases + # where custom jobInfo was stored in the project settings + for key, value in self.jobInfo.items(): + setattr(job_info, key, value) + + instance = self._instance context = instance.context - self._instance = instance - self.payload_skeleton = copy.deepcopy(payload_skeleton_template) + filepath = context.data["currentFile"] + filename = os.path.basename(filepath) - # get default deadline webservice url from deadline module - self.deadline_url = instance.context.data.get("defaultDeadline") - # if custom one is set in instance, use that - if instance.data.get("deadlineUrl"): - self.deadline_url = instance.data.get("deadlineUrl") - assert self.deadline_url, "Requires Deadline Webservice URL" + job_info.Name = "%s - %s" % (filename, instance.name) + job_info.BatchName = filename + job_info.Plugin = instance.data.get("mayaRenderPlugin", "MayaBatch") + job_info.UserName = context.data.get( + "deadlineUser", getpass.getuser()) - # just using existing names from Setting - self._job_info = self.jobInfo + # Deadline requires integers in frame range + frames = "{start}-{end}x{step}".format( + start=int(instance.data["frameStartHandle"]), + end=int(instance.data["frameEndHandle"]), + step=int(instance.data["byFrameStep"]), + ) + job_info.Frames = frames - self._plugin_info = self.pluginInfo + job_info.Pool = instance.data.get("primaryPool") + job_info.SecondaryPool = instance.data.get("secondaryPool") + job_info.ChunkSize = instance.data.get("chunkSize", 10) + job_info.Comment = context.data.get("comment") + job_info.Priority = instance.data.get("priority", self.priority) + + if self.group != "none" and self.group: + job_info.Group = self.group + + if self.limit_groups: + job_info.LimitGroups = ",".join(self.limit_groups) + + self.payload_skeleton["JobInfo"]["Name"] = jobname + self.payload_skeleton["JobInfo"]["BatchName"] = src_filename + + # Optional, enable double-click to preview rendered + # frames from Deadline Monitor + self.payload_skeleton["JobInfo"]["OutputDirectory0"] = \ + os.path.dirname(output_filename_0).replace("\\", "/") + self.payload_skeleton["JobInfo"]["OutputFilename0"] = \ + output_filename_0.replace("\\", "/") + + # Add options from RenderGlobals------------------------------------- + render_globals = instance.data.get("renderGlobals", {}) + self.payload_skeleton["JobInfo"].update(render_globals) + + keys = [ + "FTRACK_API_KEY", + "FTRACK_API_USER", + "FTRACK_SERVER", + "OPENPYPE_SG_USER", + "AVALON_PROJECT", + "AVALON_ASSET", + "AVALON_TASK", + "AVALON_APP_NAME", + "OPENPYPE_DEV", + "OPENPYPE_LOG_NO_COLORS", + "OPENPYPE_VERSION" + ] + # Add mongo url if it's enabled + if self._instance.context.data.get("deadlinePassMongoUrl"): + keys.append("OPENPYPE_MONGO") + + environment = dict({key: os.environ[key] for key in keys + if key in os.environ}, **legacy_io.Session) + + + # TODO: Taken from old publish class - test whether still needed + environment["OPENPYPE_LOG_NO_COLORS"] = "1" + environment["OPENPYPE_MAYA_VERSION"] = cmds.about(v=True) + # to recognize job from PYPE for turning Event On/Off + environment["OPENPYPE_RENDER_JOB"] = "1" + + for key in keys: + val = environment.get(key) + if val: + job_info.EnvironmentKeyValue = "{key}={value}".format( + key=key, + value=val + ) + # to recognize job from PYPE for turning Event On/Off + job_info.EnvironmentKeyValue = "OPENPYPE_RENDER_JOB=1" + + for i, filepath in enumerate(instance.data["files"]): + dirname = os.path.dirname(filepath) + fname = os.path.basename(filepath) + job_info.OutputDirectory = dirname.replace("\\", "/") + job_info.OutputFilename = fname + + # Adding file dependencies. + if self.asset_dependencies: + dependencies = instance.context.data["fileDependencies"] + dependencies.append(context.data["currentFile"]) + for dependency in dependencies: + job_info.AssetDependency = dependency + + # Add list of expected files to job + # --------------------------------- + exp = instance.data.get("expectedFiles") + + def _get_output_filename(files): + col, rem = clique.assemble(files) + if not col and rem: + # we couldn't find any collections but have + # individual files. + assert len(rem) == 1, ( + "Found multiple non related files " + "to render, don't know what to do " + "with them.") + return rem[0] + else: + return col[0].format('{head}{padding}{tail}') + + if isinstance(exp[0], dict): + # we have aovs and we need to iterate over them + for _aov, files in exp[0].items(): + output_file = _get_output_filename(files) + job_info.OutputFilename = output_file + else: + output_file = _get_output_filename(exp) + job_info.OutputFilename = output_file + + return job_info + + def get_plugin_info(self): + + instance = self._instance + context = instance.context + + renderlayer = instance.data['setMembers'] # rs_beauty + + self.payload_skeleton["PluginInfo"]["RenderLayer"] = renderlayer + self.payload_skeleton["PluginInfo"]["RenderSetupIncludeLights"] = instance.data.get("renderSetupIncludeLights") # noqa + + # Output driver to render + plugin_info = DeadlinePluginInfo( + SceneFile=context.data["currentFile"], + Version=cmds.about(version=True), + ) + + return attr.asdict(plugin_info) + + def process_submission(self): + # Override to NOT submit by default when calling super process() method + pass + + def process(self, instance): + super(MayaSubmitDeadline, self).process(instance) + + # TODO: Avoid the need for this logic here, needed for submit publish + # Store output dir for unified publisher (filesequence) + output_dir = os.path.dirname(instance.data["files"][0]) + instance.data["outputDir"] = output_dir + instance.data["toBeRenderedOn"] = "deadline" self.limit_groups = self.limit context = instance.context workspace = context.data["workspaceDir"] - anatomy = context.data['anatomy'] - instance.data["toBeRenderedOn"] = "deadline" filepath = None patches = ( @@ -336,80 +253,24 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin): "scene_patches", {}) ) - # Handle render/export from published scene or not ------------------ - if self.use_published: - patched_files = [] - for i in context: - if "workfile" not in i.data["families"]: - continue - assert i.data["publish"] is True, ( - "Workfile (scene) must be published along") - template_data = i.data.get("anatomyData") - rep = i.data.get("representations")[0].get("name") - template_data["representation"] = rep - template_data["ext"] = rep - template_data["comment"] = None - anatomy_filled = anatomy.format(template_data) - template_filled = anatomy_filled["publish"]["path"] - filepath = os.path.normpath(template_filled) - self.log.info("Using published scene for render {}".format( - filepath)) + # todo: on self.use_published originally use template_data["representation"] using .get("name") instead of .get("ext") + # todo: on self.use_published replace path for publishRenderMetadataFolder + # todo: on self.use_published apply scene patches to workfile instance + # rep = i.data.get("representations")[0].get("name") - if not os.path.exists(filepath): - self.log.error("published scene does not exist!") - raise - # now we need to switch scene in expected files - # because token will now point to published - # scene file and that might differ from current one - new_scene = os.path.splitext( - os.path.basename(filepath))[0] - orig_scene = os.path.splitext( - os.path.basename(context.data["currentFile"]))[0] - exp = instance.data.get("expectedFiles") + # if instance.data.get("publishRenderMetadataFolder"): + # instance.data["publishRenderMetadataFolder"] = \ + # instance.data["publishRenderMetadataFolder"].replace( + # orig_scene, new_scene) + # self.log.info("Scene name was switched {} -> {}".format( + # orig_scene, new_scene + # )) + # # patch workfile is needed + # if filepath not in patched_files: + # patched_file = self._patch_workfile(filepath, patches) + # patched_files.append(patched_file) - if isinstance(exp[0], dict): - # we have aovs and we need to iterate over them - new_exp = {} - for aov, files in exp[0].items(): - replaced_files = [] - for f in files: - replaced_files.append( - f.replace(orig_scene, new_scene) - ) - new_exp[aov] = replaced_files - instance.data["expectedFiles"] = [new_exp] - else: - new_exp = [] - for f in exp: - new_exp.append( - f.replace(orig_scene, new_scene) - ) - instance.data["expectedFiles"] = [new_exp] - - if instance.data.get("publishRenderMetadataFolder"): - instance.data["publishRenderMetadataFolder"] = \ - instance.data["publishRenderMetadataFolder"].replace( - orig_scene, new_scene) - self.log.info("Scene name was switched {} -> {}".format( - orig_scene, new_scene - )) - # patch workfile is needed - if filepath not in patched_files: - patched_file = self._patch_workfile(filepath, patches) - patched_files.append(patched_file) - - all_instances = [] - for result in context.data["results"]: - if (result["instance"] is not None and - result["instance"] not in all_instances): # noqa: E128 - all_instances.append(result["instance"]) - - # fallback if nothing was set - if not filepath: - self.log.warning("Falling back to workfile") - filepath = context.data["currentFile"] - - self.log.debug(filepath) + filepath = self.scene_path # collect by super().process # Gather needed data ------------------------------------------------ default_render_file = instance.context.data.get('project_settings')\ @@ -417,10 +278,8 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin): .get('RenderSettings')\ .get('default_render_image_folder') filename = os.path.basename(filepath) - comment = context.data.get("comment", "") dirname = os.path.join(workspace, default_render_file) renderlayer = instance.data['setMembers'] # rs_beauty - deadline_user = context.data.get("user", getpass.getuser()) # Always use the original work file name for the Job name even when # rendering is done from the published Work File. The original work @@ -454,116 +313,34 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin): pass # Fill in common data to payload ------------------------------------ - payload_data = {} - payload_data["filename"] = filename - payload_data["filepath"] = filepath - payload_data["jobname"] = jobname - payload_data["deadline_user"] = deadline_user - payload_data["comment"] = comment - payload_data["output_filename_0"] = output_filename_0 - payload_data["render_variables"] = render_variables - payload_data["renderlayer"] = renderlayer - payload_data["workspace"] = workspace - payload_data["dirname"] = dirname - - self.log.info("--- Submission data:") - for k, v in payload_data.items(): - self.log.info("- {}: {}".format(k, v)) - self.log.info("-" * 20) - - frame_pattern = self.payload_skeleton["JobInfo"]["Frames"] - self.payload_skeleton["JobInfo"]["Frames"] = frame_pattern.format( - start=int(self._instance.data["frameStartHandle"]), - end=int(self._instance.data["frameEndHandle"]), - step=int(self._instance.data["byFrameStep"])) - - self.payload_skeleton["JobInfo"]["Plugin"] = self._instance.data.get( - "mayaRenderPlugin", "MayaBatch") - - self.payload_skeleton["JobInfo"]["BatchName"] = src_filename - # Job name, as seen in Monitor - self.payload_skeleton["JobInfo"]["Name"] = jobname - # Arbitrary username, for visualisation in Monitor - self.payload_skeleton["JobInfo"]["UserName"] = deadline_user - # Set job priority - self.payload_skeleton["JobInfo"]["Priority"] = \ - self._instance.data.get("priority", self.priority) - - if self.group != "none" and self.group: - self.payload_skeleton["JobInfo"]["Group"] = self.group - - if self.limit_groups: - self.payload_skeleton["JobInfo"]["LimitGroups"] = \ - ",".join(self.limit_groups) - # Optional, enable double-click to preview rendered - # frames from Deadline Monitor - self.payload_skeleton["JobInfo"]["OutputDirectory0"] = \ - os.path.dirname(output_filename_0).replace("\\", "/") - self.payload_skeleton["JobInfo"]["OutputFilename0"] = \ - output_filename_0.replace("\\", "/") - - self.payload_skeleton["JobInfo"]["Comment"] = comment - self.payload_skeleton["PluginInfo"]["RenderLayer"] = renderlayer - - self.payload_skeleton["PluginInfo"]["RenderSetupIncludeLights"] = instance.data.get("renderSetupIncludeLights") # noqa - # Adding file dependencies. - dependencies = instance.context.data["fileDependencies"] - dependencies.append(filepath) - if self.asset_dependencies: - for dependency in dependencies: - key = "AssetDependency" + str(dependencies.index(dependency)) - self.payload_skeleton["JobInfo"][key] = dependency - - # Handle environments ----------------------------------------------- - # We need those to pass them to pype for it to set correct context - keys = [ - "FTRACK_API_KEY", - "FTRACK_API_USER", - "FTRACK_SERVER", - "OPENPYPE_SG_USER", - "AVALON_PROJECT", - "AVALON_ASSET", - "AVALON_TASK", - "AVALON_APP_NAME", - "OPENPYPE_DEV", - "OPENPYPE_LOG_NO_COLORS", - "OPENPYPE_VERSION" - ] - # Add mongo url if it's enabled - if instance.context.data.get("deadlinePassMongoUrl"): - keys.append("OPENPYPE_MONGO") - - environment = dict({key: os.environ[key] for key in keys - if key in os.environ}, **legacy_io.Session) - environment["OPENPYPE_LOG_NO_COLORS"] = "1" - environment["OPENPYPE_MAYA_VERSION"] = cmds.about(v=True) - # to recognize job from PYPE for turning Event On/Off - environment["OPENPYPE_RENDER_JOB"] = "1" - self.payload_skeleton["JobInfo"].update({ - "EnvironmentKeyValue%d" % index: "{key}={value}".format( - key=key, - value=environment[key] - ) for index, key in enumerate(environment) - }) - # Add options from RenderGlobals------------------------------------- - render_globals = instance.data.get("renderGlobals", {}) - self.payload_skeleton["JobInfo"].update(render_globals) + payload_data = { + "filename": filename, + "filepath": filepath, + "jobname": jobname, + "comment": comment, + "output_filename_0": output_filename_0, + "render_variables": render_variables, + "renderlayer": renderlayer, + "workspace": workspace, + "dirname": dirname, + } # Submit preceding export jobs ------------------------------------- export_job = None assert not all(x in instance.data["families"] for x in ['vrayscene', 'assscene']), ( "Vray Scene and Ass Scene options are mutually exclusive") - if "vrayscene" in instance.data["families"]: - export_job = self._submit_export(payload_data, "vray") - if "assscene" in instance.data["families"]: - export_job = self._submit_export(payload_data, "arnold") - - # Prepare main render job ------------------------------------------- if "vrayscene" in instance.data["families"]: + vray_export_payload = self._get_vray_export_payload(payload_data) + export_job = self.submit(vray_export_payload) + payload = self._get_vray_render_payload(payload_data) + elif "assscene" in instance.data["families"]: + ass_export_payload = self._get_arnold_export_payload(payload_data) + export_job = self.submit(ass_export_payload) + payload = self._get_arnold_render_payload(payload_data) else: payload = self._get_maya_payload(payload_data) @@ -572,267 +349,222 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin): if export_job: payload["JobInfo"]["JobDependency0"] = export_job - # Add list of expected files to job --------------------------------- - exp = instance.data.get("expectedFiles") - exp_index = 0 - output_filenames = {} - - if isinstance(exp[0], dict): - # we have aovs and we need to iterate over them - for _aov, files in exp[0].items(): - col, rem = clique.assemble(files) - if not col and rem: - # we couldn't find any collections but have - # individual files. - assert len(rem) == 1, ("Found multiple non related files " - "to render, don't know what to do " - "with them.") - output_file = rem[0] - if not instance.data.get("tileRendering"): - payload['JobInfo']['OutputFilename' + str(exp_index)] = output_file # noqa: E501 - else: - output_file = col[0].format('{head}{padding}{tail}') - if not instance.data.get("tileRendering"): - payload['JobInfo']['OutputFilename' + str(exp_index)] = output_file # noqa: E501 - - output_filenames['OutputFilename' + str(exp_index)] = output_file # noqa: E501 - exp_index += 1 - else: - col, rem = clique.assemble(exp) - if not col and rem: - # we couldn't find any collections but have - # individual files. - assert len(rem) == 1, ("Found multiple non related files " - "to render, don't know what to do " - "with them.") - - output_file = rem[0] - if not instance.data.get("tileRendering"): - payload['JobInfo']['OutputFilename' + str(exp_index)] = output_file # noqa: E501 - else: - output_file = col[0].format('{head}{padding}{tail}') - if not instance.data.get("tileRendering"): - payload['JobInfo']['OutputFilename' + str(exp_index)] = output_file # noqa: E501 - - output_filenames['OutputFilename' + str(exp_index)] = output_file - plugin = payload["JobInfo"]["Plugin"] self.log.info("using render plugin : {}".format(plugin)) # Store output dir for unified publisher (filesequence) instance.data["outputDir"] = os.path.dirname(output_filename_0) - self.preflight_check(instance) - # add jobInfo and pluginInfo variables from Settings - payload["JobInfo"].update(self._job_info) - payload["PluginInfo"].update(self._plugin_info) + payload["JobInfo"].update(self.jobInfo) + payload["PluginInfo"].update(self.pluginInfo) - # Prepare tiles data ------------------------------------------------ if instance.data.get("tileRendering"): - # if we have sequence of files, we need to create tile job for - # every frame + # Prepare tiles data + self._tile_render(instance, payload) + else: + # Submit main render job + self.submit(payload) - payload["JobInfo"]["TileJob"] = True - payload["JobInfo"]["TileJobTilesInX"] = instance.data.get("tilesX") - payload["JobInfo"]["TileJobTilesInY"] = instance.data.get("tilesY") - payload["PluginInfo"]["ImageHeight"] = instance.data.get("resolutionHeight") # noqa: E501 - payload["PluginInfo"]["ImageWidth"] = instance.data.get("resolutionWidth") # noqa: E501 - payload["PluginInfo"]["RegionRendering"] = True + def _tile_render(self, instance, payload): - assembly_payload = { - "AuxFiles": [], - "JobInfo": { - "BatchName": payload["JobInfo"]["BatchName"], - "Frames": 1, - "Name": "{} - Tile Assembly Job".format( - payload["JobInfo"]["Name"]), - "OutputDirectory0": - payload["JobInfo"]["OutputDirectory0"].replace( - "\\", "/"), - "Plugin": self.tile_assembler_plugin, - "MachineLimit": 1 - }, - "PluginInfo": { - "CleanupTiles": 1, - "ErrorOnMissing": True - } + # As collected by super process() + job_info = self.job_info + plugin_info = self.pluginInfo + + # if we have sequence of files, we need to create tile job for + # every frame + + job_info.TileJob = True + job_info.TileJobTilesInX = instance.data.get("tilesX") + job_info.TileJobTilesInY = instance.data.get("tilesY") + + plugin_info["ImageHeight"] = instance.data.get("resolutionHeight") + plugin_info["ImageWidth"] = instance.data.get("resolutionWidth") + plugin_info["RegionRendering"] = True + + assembly_payload = { + "AuxFiles": [], + "JobInfo": { + "BatchName": payload["JobInfo"]["BatchName"], + "Frames": 1, + "Name": "{} - Tile Assembly Job".format( + payload["JobInfo"]["Name"]), + "OutputDirectory0": + payload["JobInfo"]["OutputDirectory0"].replace( + "\\", "/"), + "Plugin": self.tile_assembler_plugin, + "MachineLimit": 1 + }, + "PluginInfo": { + "CleanupTiles": 1, + "ErrorOnMissing": True } - assembly_payload["JobInfo"].update(output_filenames) - assembly_payload["JobInfo"]["Priority"] = self._instance.data.get( - "tile_priority", self.tile_priority) - assembly_payload["JobInfo"]["UserName"] = deadline_user + } + assembly_payload["JobInfo"]["Priority"] = self._instance.data.get( + "tile_priority", self.tile_priority) - frame_payloads = [] - assembly_payloads = [] + frame_payloads = [] + assembly_payloads = [] - R_FRAME_NUMBER = re.compile(r".+\.(?P[0-9]+)\..+") # noqa: N806, E501 - REPL_FRAME_NUMBER = re.compile(r"(.+\.)([0-9]+)(\..+)") # noqa: N806, E501 + R_FRAME_NUMBER = re.compile( + r".+\.(?P[0-9]+)\..+") # noqa: N806, E501 + REPL_FRAME_NUMBER = re.compile( + r"(.+\.)([0-9]+)(\..+)") # noqa: N806, E501 - if isinstance(exp[0], dict): - # we have aovs and we need to iterate over them - # get files from `beauty` - files = exp[0].get("beauty") - # assembly files are used for assembly jobs as we need to put - # together all AOVs - assembly_files = list( - itertools.chain.from_iterable( - [f for _, f in exp[0].items()])) - if not files: - # if beauty doesn't exists, use first aov we found - files = exp[0].get(list(exp[0].keys())[0]) - else: - files = exp - assembly_files = files + exp = instance.data["expectedFiles"] + if isinstance(exp[0], dict): + # we have aovs and we need to iterate over them + # get files from `beauty` + files = exp[0].get("beauty") + # assembly files are used for assembly jobs as we need to put + # together all AOVs + assembly_files = list( + itertools.chain.from_iterable( + [f for _, f in exp[0].items()])) + if not files: + # if beauty doesn't exists, use first aov we found + files = exp[0].get(list(exp[0].keys())[0]) + else: + files = exp + assembly_files = files - frame_jobs = {} + frame_jobs = {} - file_index = 1 - for file in files: - frame = re.search(R_FRAME_NUMBER, file).group("frame") - new_payload = copy.deepcopy(payload) - new_payload["JobInfo"]["Name"] = \ - "{} (Frame {} - {} tiles)".format( - payload["JobInfo"]["Name"], - frame, - instance.data.get("tilesX") * instance.data.get("tilesY") # noqa: E501 + file_index = 1 + for file in files: + frame = re.search(R_FRAME_NUMBER, file).group("frame") + new_payload = copy.deepcopy(payload) + new_payload["JobInfo"]["Name"] = \ + "{} (Frame {} - {} tiles)".format( + payload["JobInfo"]["Name"], + frame, + instance.data.get("tilesX") * instance.data.get("tilesY") + # noqa: E501 ) - self.log.info( - "... preparing job {}".format( - new_payload["JobInfo"]["Name"])) - new_payload["JobInfo"]["TileJobFrame"] = frame + self.log.info( + "... preparing job {}".format( + new_payload["JobInfo"]["Name"])) + new_payload["JobInfo"]["TileJobFrame"] = frame - tiles_data = _format_tiles( + tiles_data = _format_tiles( + file, 0, + instance.data.get("tilesX"), + instance.data.get("tilesY"), + instance.data.get("resolutionWidth"), + instance.data.get("resolutionHeight"), + payload["PluginInfo"]["OutputFilePrefix"] + )[0] + new_payload["JobInfo"].update(tiles_data["JobInfo"]) + new_payload["PluginInfo"].update(tiles_data["PluginInfo"]) + + self.log.info("hashing {} - {}".format(file_index, file)) + job_hash = hashlib.sha256( + ("{}_{}".format(file_index, file)).encode("utf-8")) + frame_jobs[frame] = job_hash.hexdigest() + new_payload["JobInfo"]["ExtraInfo0"] = job_hash.hexdigest() + new_payload["JobInfo"]["ExtraInfo1"] = file + + frame_payloads.append(new_payload) + file_index += 1 + + file_index = 1 + for file in assembly_files: + frame = re.search(R_FRAME_NUMBER, file).group("frame") + new_assembly_payload = copy.deepcopy(assembly_payload) + new_assembly_payload["JobInfo"]["Name"] = \ + "{} (Frame {})".format( + assembly_payload["JobInfo"]["Name"], + frame) + new_assembly_payload["JobInfo"]["OutputFilename0"] = re.sub( + REPL_FRAME_NUMBER, + "\\1{}\\3".format("#" * len(frame)), file) + + new_assembly_payload["PluginInfo"]["Renderer"] = \ + self._instance.data["renderer"] # noqa: E501 + new_assembly_payload["JobInfo"]["ExtraInfo0"] = frame_jobs[ + frame] # noqa: E501 + new_assembly_payload["JobInfo"]["ExtraInfo1"] = file + assembly_payloads.append(new_assembly_payload) + file_index += 1 + + self.log.info( + "Submitting tile job(s) [{}] ...".format(len(frame_payloads))) + + url = "{}/api/jobs".format(self.deadline_url) + tiles_count = instance.data.get("tilesX") * instance.data.get( + "tilesY") # noqa: E501 + + for tile_job in frame_payloads: + response = self.submit(tile_job) + + job_id = response.json()["_id"] + hash = response.json()["Props"]["Ex0"] + + for assembly_job in assembly_payloads: + if assembly_job["JobInfo"]["ExtraInfo0"] == hash: + assembly_job["JobInfo"]["JobDependency0"] = job_id + + for assembly_job in assembly_payloads: + file = assembly_job["JobInfo"]["ExtraInfo1"] + # write assembly job config files + now = datetime.now() + + config_file = os.path.join( + os.path.dirname(output_filename_0), + "{}_config_{}.txt".format( + os.path.splitext(file)[0], + now.strftime("%Y_%m_%d_%H_%M_%S") + ) + ) + + try: + if not os.path.isdir(os.path.dirname(config_file)): + os.makedirs(os.path.dirname(config_file)) + except OSError: + # directory is not available + self.log.warning( + "Path is unreachable: `{}`".format( + os.path.dirname(config_file))) + + # add config file as job auxFile + assembly_job["AuxFiles"] = [config_file] + + with open(config_file, "w") as cf: + print("TileCount={}".format(tiles_count), file=cf) + print("ImageFileName={}".format(file), file=cf) + print("ImageWidth={}".format( + instance.data.get("resolutionWidth")), file=cf) + print("ImageHeight={}".format( + instance.data.get("resolutionHeight")), file=cf) + + tiles = _format_tiles( file, 0, instance.data.get("tilesX"), instance.data.get("tilesY"), instance.data.get("resolutionWidth"), instance.data.get("resolutionHeight"), payload["PluginInfo"]["OutputFilePrefix"] - )[0] - new_payload["JobInfo"].update(tiles_data["JobInfo"]) - new_payload["PluginInfo"].update(tiles_data["PluginInfo"]) + )[1] + sorted(tiles) + for k, v in tiles.items(): + print("{}={}".format(k, v), file=cf) - self.log.info("hashing {} - {}".format(file_index, file)) - job_hash = hashlib.sha256( - ("{}_{}".format(file_index, file)).encode("utf-8")) - frame_jobs[frame] = job_hash.hexdigest() - new_payload["JobInfo"]["ExtraInfo0"] = job_hash.hexdigest() - new_payload["JobInfo"]["ExtraInfo1"] = file + job_idx = 1 + instance.data["assemblySubmissionJobs"] = [] + for ass_job in assembly_payloads: + self.log.info("submitting assembly job {} of {}".format( + job_idx, len(assembly_payloads) + )) + response = self.submit(ass_job) - frame_payloads.append(new_payload) - file_index += 1 + instance.data["assemblySubmissionJobs"].append( + response.json()["_id"]) + job_idx += 1 - file_index = 1 - for file in assembly_files: - frame = re.search(R_FRAME_NUMBER, file).group("frame") - new_assembly_payload = copy.deepcopy(assembly_payload) - new_assembly_payload["JobInfo"]["Name"] = \ - "{} (Frame {})".format( - assembly_payload["JobInfo"]["Name"], - frame) - new_assembly_payload["JobInfo"]["OutputFilename0"] = re.sub( - REPL_FRAME_NUMBER, - "\\1{}\\3".format("#" * len(frame)), file) - - new_assembly_payload["PluginInfo"]["Renderer"] = self._instance.data["renderer"] # noqa: E501 - new_assembly_payload["JobInfo"]["ExtraInfo0"] = frame_jobs[frame] # noqa: E501 - new_assembly_payload["JobInfo"]["ExtraInfo1"] = file - assembly_payloads.append(new_assembly_payload) - file_index += 1 - - self.log.info( - "Submitting tile job(s) [{}] ...".format(len(frame_payloads))) - - url = "{}/api/jobs".format(self.deadline_url) - tiles_count = instance.data.get("tilesX") * instance.data.get("tilesY") # noqa: E501 - - for tile_job in frame_payloads: - response = requests_post(url, json=tile_job) - if not response.ok: - raise Exception(response.text) - - job_id = response.json()["_id"] - hash = response.json()["Props"]["Ex0"] - - for assembly_job in assembly_payloads: - if assembly_job["JobInfo"]["ExtraInfo0"] == hash: - assembly_job["JobInfo"]["JobDependency0"] = job_id - - for assembly_job in assembly_payloads: - file = assembly_job["JobInfo"]["ExtraInfo1"] - # write assembly job config files - now = datetime.now() - - config_file = os.path.join( - os.path.dirname(output_filename_0), - "{}_config_{}.txt".format( - os.path.splitext(file)[0], - now.strftime("%Y_%m_%d_%H_%M_%S") - ) - ) - - try: - if not os.path.isdir(os.path.dirname(config_file)): - os.makedirs(os.path.dirname(config_file)) - except OSError: - # directory is not available - self.log.warning( - "Path is unreachable: `{}`".format( - os.path.dirname(config_file))) - - # add config file as job auxFile - assembly_job["AuxFiles"] = [config_file] - - with open(config_file, "w") as cf: - print("TileCount={}".format(tiles_count), file=cf) - print("ImageFileName={}".format(file), file=cf) - print("ImageWidth={}".format( - instance.data.get("resolutionWidth")), file=cf) - print("ImageHeight={}".format( - instance.data.get("resolutionHeight")), file=cf) - - tiles = _format_tiles( - file, 0, - instance.data.get("tilesX"), - instance.data.get("tilesY"), - instance.data.get("resolutionWidth"), - instance.data.get("resolutionHeight"), - payload["PluginInfo"]["OutputFilePrefix"] - )[1] - sorted(tiles) - for k, v in tiles.items(): - print("{}={}".format(k, v), file=cf) - - job_idx = 1 - instance.data["assemblySubmissionJobs"] = [] - for ass_job in assembly_payloads: - self.log.info("submitting assembly job {} of {}".format( - job_idx, len(assembly_payloads) - )) - self.log.debug(json.dumps(ass_job, indent=4, sort_keys=True)) - response = requests_post(url, json=ass_job) - if not response.ok: - raise Exception(response.text) - - instance.data["assemblySubmissionJobs"].append( - response.json()["_id"]) - job_idx += 1 - - instance.data["jobBatchName"] = payload["JobInfo"]["BatchName"] - self.log.info("Setting batch name on instance: {}".format( - instance.data["jobBatchName"])) - else: - # Submit job to farm -------------------------------------------- - self.log.info("Submitting ...") - self.log.debug(json.dumps(payload, indent=4, sort_keys=True)) - - # E.g. http://192.168.0.1:8082/api/jobs - url = "{}/api/jobs".format(self.deadline_url) - response = requests_post(url, json=payload) - if not response.ok: - raise Exception(response.text) - instance.data["deadlineSubmissionJob"] = response.json() + instance.data["jobBatchName"] = payload["JobInfo"]["BatchName"] + self.log.info("Setting batch name on instance: {}".format( + instance.data["jobBatchName"])) def _get_maya_payload(self, data): payload = copy.deepcopy(self.payload_skeleton) @@ -1045,39 +777,6 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin): payload["PluginInfo"].update(plugin_info) return payload - def _submit_export(self, data, format): - if format == "vray": - payload = self._get_vray_export_payload(data) - self.log.info("Submitting vrscene export job.") - elif format == "arnold": - payload = self._get_arnold_export_payload(data) - self.log.info("Submitting ass export job.") - - url = "{}/api/jobs".format(self.deadline_url) - response = requests_post(url, json=payload) - if not response.ok: - self.log.error("Submition failed!") - self.log.error(response.status_code) - self.log.error(response.content) - self.log.debug(payload) - raise RuntimeError(response.text) - - dependency = response.json() - return dependency["_id"] - - def preflight_check(self, instance): - """Ensure the startFrame, endFrame and byFrameStep are integers.""" - for key in ("frameStartHandle", "frameEndHandle", "byFrameStep"): - value = instance.data[key] - - if int(value) == value: - continue - - self.log.warning( - "%f=%d was rounded off to nearest integer" - % (value, int(value)) - ) - def format_vray_output_filename(self, filename, template, dir=False): """Format the expected output file of the Export job. @@ -1160,3 +859,209 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin): "Applied {} patch to scene.".format( patches[i]["name"])) return file + + +def _format_tiles( + filename, index, tiles_x, tiles_y, + width, height, prefix): + """Generate tile entries for Deadline tile job. + + Returns two dictionaries - one that can be directly used in Deadline + job, second that can be used for Deadline Assembly job configuration + file. + + This will format tile names: + + Example:: + { + "OutputFilename0Tile0": "_tile_1x1_4x4_Main_beauty.1001.exr", + "OutputFilename0Tile1": "_tile_2x1_4x4_Main_beauty.1001.exr" + } + + And add tile prefixes like: + + Example:: + Image prefix is: + `maya///_` + + Result for tile 0 for 4x4 will be: + `maya///_tile_1x1_4x4__` + + Calculating coordinates is tricky as in Job they are defined as top, + left, bottom, right with zero being in top-left corner. But Assembler + configuration file takes tile coordinates as X, Y, Width and Height and + zero is bottom left corner. + + Args: + filename (str): Filename to process as tiles. + index (int): Index of that file if it is sequence. + tiles_x (int): Number of tiles in X. + tiles_y (int): Number if tikes in Y. + width (int): Width resolution of final image. + height (int): Height resolution of final image. + prefix (str): Image prefix. + + Returns: + (dict, dict): Tuple of two dictionaires - first can be used to + extend JobInfo, second has tiles x, y, width and height + used for assembler configuration. + + """ + tile = 0 + out = {"JobInfo": {}, "PluginInfo": {}} + cfg = OrderedDict() + w_space = width / tiles_x + h_space = height / tiles_y + + cfg["TilesCropped"] = "False" + + for tile_x in range(1, tiles_x + 1): + for tile_y in reversed(range(1, tiles_y + 1)): + tile_prefix = "_tile_{}x{}_{}x{}_".format( + tile_x, tile_y, + tiles_x, + tiles_y + ) + out_tile_index = "OutputFilename{}Tile{}".format( + str(index), tile + ) + new_filename = "{}/{}{}".format( + os.path.dirname(filename), + tile_prefix, + os.path.basename(filename) + ) + out["JobInfo"][out_tile_index] = new_filename + out["PluginInfo"]["RegionPrefix{}".format(str(tile))] = \ + "/{}".format(tile_prefix).join(prefix.rsplit("/", 1)) + + out["PluginInfo"]["RegionTop{}".format(tile)] = int(height) - (tile_y * h_space) # noqa: E501 + out["PluginInfo"]["RegionBottom{}".format(tile)] = int(height) - ((tile_y - 1) * h_space) - 1 # noqa: E501 + out["PluginInfo"]["RegionLeft{}".format(tile)] = (tile_x - 1) * w_space # noqa: E501 + out["PluginInfo"]["RegionRight{}".format(tile)] = (tile_x * w_space) - 1 # noqa: E501 + + cfg["Tile{}".format(tile)] = new_filename + cfg["Tile{}Tile".format(tile)] = new_filename + cfg["Tile{}FileName".format(tile)] = new_filename + cfg["Tile{}X".format(tile)] = (tile_x - 1) * w_space + + cfg["Tile{}Y".format(tile)] = int(height) - (tile_y * h_space) + + cfg["Tile{}Width".format(tile)] = w_space + cfg["Tile{}Height".format(tile)] = h_space + + tile += 1 + return out, cfg + + +def get_renderer_variables(renderlayer, root): + """Retrieve the extension which has been set in the VRay settings. + + Will return None if the current renderer is not VRay + For Maya 2016.5 and up the renderSetup creates renderSetupLayer node which + start with `rs`. Use the actual node name, do NOT use the `nice name` + + Args: + renderlayer (str): the node name of the renderlayer. + root (str): base path to render + + Returns: + dict + + """ + renderer = lib.get_renderer(renderlayer or lib.get_current_renderlayer()) + render_attrs = lib.RENDER_ATTRS.get(renderer, lib.RENDER_ATTRS["default"]) + + padding = cmds.getAttr("{}.{}".format(render_attrs["node"], + render_attrs["padding"])) + + filename_0 = cmds.renderSettings( + fullPath=True, + gin="#" * int(padding), + lut=True, + layer=renderlayer or lib.get_current_renderlayer())[0] + filename_0 = re.sub('_', '_beauty', + filename_0, flags=re.IGNORECASE) + prefix_attr = "defaultRenderGlobals.imageFilePrefix" + + scene = cmds.file(query=True, sceneName=True) + scene, _ = os.path.splitext(os.path.basename(scene)) + + if renderer == "vray": + renderlayer = renderlayer.split("_")[-1] + # Maya's renderSettings function does not return V-Ray file extension + # so we get the extension from vraySettings + extension = cmds.getAttr("vraySettings.imageFormatStr") + + # When V-Ray image format has not been switched once from default .png + # the getAttr command above returns None. As such we explicitly set + # it to `.png` + if extension is None: + extension = "png" + + if extension in ["exr (multichannel)", "exr (deep)"]: + extension = "exr" + + prefix_attr = "vraySettings.fileNamePrefix" + filename_prefix = cmds.getAttr(prefix_attr) + # we need to determine path for vray as maya `renderSettings` query + # does not work for vray. + + filename_0 = re.sub('', scene, filename_prefix, flags=re.IGNORECASE) # noqa: E501 + filename_0 = re.sub('', renderlayer, filename_0, flags=re.IGNORECASE) # noqa: E501 + filename_0 = "{}.{}.{}".format( + filename_0, "#" * int(padding), extension) + filename_0 = os.path.normpath(os.path.join(root, filename_0)) + elif renderer == "renderman": + prefix_attr = "rmanGlobals.imageFileFormat" + # NOTE: This is guessing extensions from renderman display types. + # Some of them are just framebuffers, d_texture format can be + # set in display setting. We set those now to None, but it + # should be handled more gracefully. + display_types = { + "d_deepexr": "exr", + "d_it": None, + "d_null": None, + "d_openexr": "exr", + "d_png": "png", + "d_pointcloud": "ptc", + "d_targa": "tga", + "d_texture": None, + "d_tiff": "tif" + } + + extension = display_types.get( + cmds.listConnections("rmanDefaultDisplay.displayType")[0], + "exr" + ) or "exr" + + filename_prefix = "{}/{}".format( + cmds.getAttr("rmanGlobals.imageOutputDir"), + cmds.getAttr("rmanGlobals.imageFileFormat") + ) + + renderlayer = renderlayer.split("_")[-1] + + filename_0 = re.sub('', scene, filename_prefix, flags=re.IGNORECASE) # noqa: E501 + filename_0 = re.sub('', renderlayer, filename_0, flags=re.IGNORECASE) # noqa: E501 + filename_0 = re.sub('', "#" * int(padding), filename_0, flags=re.IGNORECASE) # noqa: E501 + filename_0 = re.sub('', extension, filename_0, flags=re.IGNORECASE) # noqa: E501 + filename_0 = os.path.normpath(os.path.join(root, filename_0)) + elif renderer == "redshift": + # mapping redshift extension dropdown values to strings + ext_mapping = ["iff", "exr", "tif", "png", "tga", "jpg"] + extension = ext_mapping[ + cmds.getAttr("redshiftOptions.imageFormat") + ] + else: + # Get the extension, getAttr defaultRenderGlobals.imageFormat + # returns an index number. + filename_base = os.path.basename(filename_0) + extension = os.path.splitext(filename_base)[-1].strip(".") + + filename_prefix = cmds.getAttr(prefix_attr) + return {"ext": extension, + "filename_prefix": filename_prefix, + "padding": padding, + "filename_0": filename_0} + + From aaa95efb3c80080166de20c9f48bbfe6a8c11548 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 24 Aug 2022 13:45:53 +0200 Subject: [PATCH 014/122] Removed submodule vendor/configs/OpenColorIO-Configs --- vendor/configs/OpenColorIO-Configs | 1 - 1 file changed, 1 deletion(-) delete mode 160000 vendor/configs/OpenColorIO-Configs diff --git a/vendor/configs/OpenColorIO-Configs b/vendor/configs/OpenColorIO-Configs deleted file mode 160000 index 0bb079c08b..0000000000 --- a/vendor/configs/OpenColorIO-Configs +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 0bb079c08be410030669cbf5f19ff869b88af953 From 4c16b8930fd38d6c9f792c5983d6e2e88f1587ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Je=C5=BEek?= Date: Wed, 24 Aug 2022 14:14:47 +0200 Subject: [PATCH 015/122] Update openpype/plugins/publish/collect_audio.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- openpype/plugins/publish/collect_audio.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/openpype/plugins/publish/collect_audio.py b/openpype/plugins/publish/collect_audio.py index e2fb766ec4..0825c281ad 100644 --- a/openpype/plugins/publish/collect_audio.py +++ b/openpype/plugins/publish/collect_audio.py @@ -66,7 +66,10 @@ class CollectAudio(pyblish.api.InstancePlugin): self.log.info("Audio Data added to instance ...") def _get_repre_doc(self, instance): - cache = instance.context.data.get("__cache_asset_audio", {}) + cache = instance.context.data.get("__cache_asset_audio") + if cache is None: + cache = {} + instance.context.data["__cache_asset_audio"] = cache asset_name = instance.data["asset"] # first try to get it from cache From 6ff16a6d25f2b81f5634d526059734225c556ede Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Je=C5=BEek?= Date: Wed, 24 Aug 2022 14:14:58 +0200 Subject: [PATCH 016/122] Update openpype/plugins/publish/collect_audio.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- openpype/plugins/publish/collect_audio.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openpype/plugins/publish/collect_audio.py b/openpype/plugins/publish/collect_audio.py index 0825c281ad..7d53b24e54 100644 --- a/openpype/plugins/publish/collect_audio.py +++ b/openpype/plugins/publish/collect_audio.py @@ -101,6 +101,5 @@ class CollectAudio(pyblish.api.InstancePlugin): # update cache cache[asset_name] = repre_doc - instance.context.data["__cache_asset_audio"].update(cache) return repre_doc From da5353aa2dd1d9e774a72dbc57a6ac5d7368afdd Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 24 Aug 2022 14:20:35 +0200 Subject: [PATCH 017/122] git: update gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index ea5b20eb69..4b773e97ed 100644 --- a/.gitignore +++ b/.gitignore @@ -107,3 +107,6 @@ website/.docusaurus mypy.ini tools/run_eventserver.* + +# Developer tools +tools/dev_* From 0f95f87d773ddcbe979fe28d5f0196f1befad38e Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Wed, 24 Aug 2022 14:59:14 +0200 Subject: [PATCH 018/122] More draft refactoring - still not functional (WIP commit for my own sanity) --- .../plugins/publish/submit_maya_deadline.py | 185 ++++-------------- 1 file changed, 35 insertions(+), 150 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 6dfa48a9f8..5a7d0b98c6 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -31,7 +31,6 @@ import clique from maya import cmds -from openpype.hosts.maya.api import lib from openpype.pipeline import legacy_io from openpype_modules.deadline import abstract_submit_deadline @@ -87,11 +86,15 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): instance = self._instance context = instance.context - filepath = context.data["currentFile"] - filename = os.path.basename(filepath) + # Always use the original work file name for the Job name even when + # rendering is done from the published Work File. The original work + # file name is clearer because it can also have subversion strings, + # etc. which are stripped for the published file. + src_filepath = context.data["currentFile"] + src_filename = os.path.basename(src_filepath) - job_info.Name = "%s - %s" % (filename, instance.name) - job_info.BatchName = filename + job_info.Name = "%s - %s" % (src_filename, instance.name) + job_info.BatchName = src_filename job_info.Plugin = instance.data.get("mayaRenderPlugin", "MayaBatch") job_info.UserName = context.data.get( "deadlineUser", getpass.getuser()) @@ -116,9 +119,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): if self.limit_groups: job_info.LimitGroups = ",".join(self.limit_groups) - self.payload_skeleton["JobInfo"]["Name"] = jobname - self.payload_skeleton["JobInfo"]["BatchName"] = src_filename - # Optional, enable double-click to preview rendered # frames from Deadline Monitor self.payload_skeleton["JobInfo"]["OutputDirectory0"] = \ @@ -227,11 +227,17 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): return attr.asdict(plugin_info) def process_submission(self): - # Override to NOT submit by default when calling super process() method - pass - def process(self, instance): - super(MayaSubmitDeadline, self).process(instance) + instance = self._instance + context = instance.context + + # Generated by AbstractSubmitDeadline. The `job_info`, `plugin_info` + # and `aux_files` are the skeleton payloads that are the basis for + # all the maya submissions + job_info = self.job_info + plugin_info = self.plugin_info + aux_files = self.aux_files + filepath = self.scene_path # publish if `use_publish` else workfile # TODO: Avoid the need for this logic here, needed for submit publish # Store output dir for unified publisher (filesequence) @@ -241,21 +247,19 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): self.limit_groups = self.limit - context = instance.context - workspace = context.data["workspaceDir"] - - filepath = None - patches = ( - context.data["project_settings"].get( - "deadline", {}).get( - "publish", {}).get( - "MayaSubmitDeadline", {}).get( - "scene_patches", {}) - ) + # Patch workfile (only when use_published is enabled) + if self.use_published: + patches = ( + context.data["project_settings"].get( + "deadline", {}).get( + "publish", {}).get( + "MayaSubmitDeadline", {}).get( + "scene_patches", {}) + ) + self._patch_workfile(filepath, patches) # todo: on self.use_published originally use template_data["representation"] using .get("name") instead of .get("ext") # todo: on self.use_published replace path for publishRenderMetadataFolder - # todo: on self.use_published apply scene patches to workfile instance # rep = i.data.get("representations")[0].get("name") # if instance.data.get("publishRenderMetadataFolder"): @@ -270,9 +274,8 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): # patched_file = self._patch_workfile(filepath, patches) # patched_files.append(patched_file) - filepath = self.scene_path # collect by super().process - # Gather needed data ------------------------------------------------ + workspace = context.data["workspaceDir"] default_render_file = instance.context.data.get('project_settings')\ .get('maya')\ .get('RenderSettings')\ @@ -281,14 +284,10 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): dirname = os.path.join(workspace, default_render_file) renderlayer = instance.data['setMembers'] # rs_beauty - # Always use the original work file name for the Job name even when - # rendering is done from the published Work File. The original work - # file name is clearer because it can also have subversion strings, - # etc. which are stripped for the published file. - src_filename = os.path.basename(context.data["currentFile"]) - jobname = "%s - %s" % (src_filename, instance.name) - # Get the variables depending on the renderer + # TODO: Find replacement logic for `get_renderer_variables` through + # what is collected for the render or is implemented in maya + # api `lib_renderproducts` render_variables = get_renderer_variables(renderlayer, dirname) filename_0 = render_variables["filename_0"] if self.use_published: @@ -842,8 +841,8 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): str: Patched file path or None """ - if os.path.splitext(file)[1].lower() != ".ma" or not patches: - return None + if not patches or os.path.splitext(file)[1].lower() != ".ma": + return compiled_regex = [re.compile(p["regex"]) for p in patches] with open(file, "r+") as pf: @@ -931,7 +930,7 @@ def _format_tiles( os.path.basename(filename) ) out["JobInfo"][out_tile_index] = new_filename - out["PluginInfo"]["RegionPrefix{}".format(str(tile))] = \ + out["PluginInfo"]["RegionPrefix{}".format(tile)] = \ "/{}".format(tile_prefix).join(prefix.rsplit("/", 1)) out["PluginInfo"]["RegionTop{}".format(tile)] = int(height) - (tile_y * h_space) # noqa: E501 @@ -951,117 +950,3 @@ def _format_tiles( tile += 1 return out, cfg - - -def get_renderer_variables(renderlayer, root): - """Retrieve the extension which has been set in the VRay settings. - - Will return None if the current renderer is not VRay - For Maya 2016.5 and up the renderSetup creates renderSetupLayer node which - start with `rs`. Use the actual node name, do NOT use the `nice name` - - Args: - renderlayer (str): the node name of the renderlayer. - root (str): base path to render - - Returns: - dict - - """ - renderer = lib.get_renderer(renderlayer or lib.get_current_renderlayer()) - render_attrs = lib.RENDER_ATTRS.get(renderer, lib.RENDER_ATTRS["default"]) - - padding = cmds.getAttr("{}.{}".format(render_attrs["node"], - render_attrs["padding"])) - - filename_0 = cmds.renderSettings( - fullPath=True, - gin="#" * int(padding), - lut=True, - layer=renderlayer or lib.get_current_renderlayer())[0] - filename_0 = re.sub('_', '_beauty', - filename_0, flags=re.IGNORECASE) - prefix_attr = "defaultRenderGlobals.imageFilePrefix" - - scene = cmds.file(query=True, sceneName=True) - scene, _ = os.path.splitext(os.path.basename(scene)) - - if renderer == "vray": - renderlayer = renderlayer.split("_")[-1] - # Maya's renderSettings function does not return V-Ray file extension - # so we get the extension from vraySettings - extension = cmds.getAttr("vraySettings.imageFormatStr") - - # When V-Ray image format has not been switched once from default .png - # the getAttr command above returns None. As such we explicitly set - # it to `.png` - if extension is None: - extension = "png" - - if extension in ["exr (multichannel)", "exr (deep)"]: - extension = "exr" - - prefix_attr = "vraySettings.fileNamePrefix" - filename_prefix = cmds.getAttr(prefix_attr) - # we need to determine path for vray as maya `renderSettings` query - # does not work for vray. - - filename_0 = re.sub('', scene, filename_prefix, flags=re.IGNORECASE) # noqa: E501 - filename_0 = re.sub('', renderlayer, filename_0, flags=re.IGNORECASE) # noqa: E501 - filename_0 = "{}.{}.{}".format( - filename_0, "#" * int(padding), extension) - filename_0 = os.path.normpath(os.path.join(root, filename_0)) - elif renderer == "renderman": - prefix_attr = "rmanGlobals.imageFileFormat" - # NOTE: This is guessing extensions from renderman display types. - # Some of them are just framebuffers, d_texture format can be - # set in display setting. We set those now to None, but it - # should be handled more gracefully. - display_types = { - "d_deepexr": "exr", - "d_it": None, - "d_null": None, - "d_openexr": "exr", - "d_png": "png", - "d_pointcloud": "ptc", - "d_targa": "tga", - "d_texture": None, - "d_tiff": "tif" - } - - extension = display_types.get( - cmds.listConnections("rmanDefaultDisplay.displayType")[0], - "exr" - ) or "exr" - - filename_prefix = "{}/{}".format( - cmds.getAttr("rmanGlobals.imageOutputDir"), - cmds.getAttr("rmanGlobals.imageFileFormat") - ) - - renderlayer = renderlayer.split("_")[-1] - - filename_0 = re.sub('', scene, filename_prefix, flags=re.IGNORECASE) # noqa: E501 - filename_0 = re.sub('', renderlayer, filename_0, flags=re.IGNORECASE) # noqa: E501 - filename_0 = re.sub('', "#" * int(padding), filename_0, flags=re.IGNORECASE) # noqa: E501 - filename_0 = re.sub('', extension, filename_0, flags=re.IGNORECASE) # noqa: E501 - filename_0 = os.path.normpath(os.path.join(root, filename_0)) - elif renderer == "redshift": - # mapping redshift extension dropdown values to strings - ext_mapping = ["iff", "exr", "tif", "png", "tga", "jpg"] - extension = ext_mapping[ - cmds.getAttr("redshiftOptions.imageFormat") - ] - else: - # Get the extension, getAttr defaultRenderGlobals.imageFormat - # returns an index number. - filename_base = os.path.basename(filename_0) - extension = os.path.splitext(filename_base)[-1].strip(".") - - filename_prefix = cmds.getAttr(prefix_attr) - return {"ext": extension, - "filename_prefix": filename_prefix, - "padding": padding, - "filename_0": filename_0} - - From 67fd21edae5d7c3670ee704558ff10505c64a783 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 15:40:32 +0200 Subject: [PATCH 019/122] imlemented fusion addon --- openpype/hosts/fusion/__init__.py | 10 ++++++++++ openpype/hosts/fusion/addon.py | 23 +++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 openpype/hosts/fusion/addon.py diff --git a/openpype/hosts/fusion/__init__.py b/openpype/hosts/fusion/__init__.py index e69de29bb2..ddae01890b 100644 --- a/openpype/hosts/fusion/__init__.py +++ b/openpype/hosts/fusion/__init__.py @@ -0,0 +1,10 @@ +from .addon import ( + FusionAddon, + FUSION_HOST_DIR, +) + + +__all__ = ( + "FusionAddon", + "FUSION_HOST_DIR", +) diff --git a/openpype/hosts/fusion/addon.py b/openpype/hosts/fusion/addon.py new file mode 100644 index 0000000000..97fb262517 --- /dev/null +++ b/openpype/hosts/fusion/addon.py @@ -0,0 +1,23 @@ +import os +from openpype.modules import OpenPypeModule +from openpype.modules.interfaces import IHostModule + +FUSION_HOST_DIR = os.path.dirname(os.path.abspath(__file__)) + + +class FusionAddon(OpenPypeModule, IHostModule): + name = "fusion" + host_name = "fusion" + + def initialize(self, module_settings): + self.enabled = True + + def get_launch_hook_paths(self, app): + if app.host_name != self.host_name: + return [] + return [ + os.path.join(FUSION_HOST_DIR, "hooks") + ] + + def get_workfile_extensions(self): + return [".comp"] From 3ea7510693ec23623ec7760f3479e10b363a2adf Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 15:40:40 +0200 Subject: [PATCH 020/122] removed usage of HOST_WORKFILE_EXTENSIONS --- openpype/hosts/fusion/api/workio.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/openpype/hosts/fusion/api/workio.py b/openpype/hosts/fusion/api/workio.py index a1710c6e3a..89752d3e6d 100644 --- a/openpype/hosts/fusion/api/workio.py +++ b/openpype/hosts/fusion/api/workio.py @@ -2,13 +2,11 @@ import sys import os -from openpype.pipeline import HOST_WORKFILE_EXTENSIONS - from .pipeline import get_current_comp def file_extensions(): - return HOST_WORKFILE_EXTENSIONS["fusion"] + return [".comp"] def has_unsaved_changes(): From cf50d1dd1f00fe178d8bfc28a039579222cab1d6 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 15:40:56 +0200 Subject: [PATCH 021/122] reuse 'FUSION_HOST_DIR' from fusion public api --- openpype/hosts/fusion/api/pipeline.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/openpype/hosts/fusion/api/pipeline.py b/openpype/hosts/fusion/api/pipeline.py index 54a6c94b60..987eae214b 100644 --- a/openpype/hosts/fusion/api/pipeline.py +++ b/openpype/hosts/fusion/api/pipeline.py @@ -18,12 +18,11 @@ from openpype.pipeline import ( deregister_inventory_action_path, AVALON_CONTAINER_ID, ) -import openpype.hosts.fusion +from openpype.hosts.fusion import FUSION_HOST_DIR log = Logger.get_logger(__name__) -HOST_DIR = os.path.dirname(os.path.abspath(openpype.hosts.fusion.__file__)) -PLUGINS_DIR = os.path.join(HOST_DIR, "plugins") +PLUGINS_DIR = os.path.join(FUSION_HOST_DIR, "plugins") PUBLISH_PATH = os.path.join(PLUGINS_DIR, "publish") LOAD_PATH = os.path.join(PLUGINS_DIR, "load") From 38932ba3012668e8ca2a239ffa2b68b002d979d6 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Sat, 27 Aug 2022 10:38:01 +0200 Subject: [PATCH 022/122] fixed interface name --- openpype/hosts/fusion/addon.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/fusion/addon.py b/openpype/hosts/fusion/addon.py index 97fb262517..e257005061 100644 --- a/openpype/hosts/fusion/addon.py +++ b/openpype/hosts/fusion/addon.py @@ -1,11 +1,11 @@ import os from openpype.modules import OpenPypeModule -from openpype.modules.interfaces import IHostModule +from openpype.modules.interfaces import IHostAddon FUSION_HOST_DIR = os.path.dirname(os.path.abspath(__file__)) -class FusionAddon(OpenPypeModule, IHostModule): +class FusionAddon(OpenPypeModule, IHostAddon): name = "fusion" host_name = "fusion" From 0d8cf12618cee76a5e144429a3459074b14e4adf Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 29 Aug 2022 17:14:22 +0200 Subject: [PATCH 023/122] define new source where publish templates are not defined in integrate plubin --- .../defaults/project_settings/global.json | 3 + .../defaults/project_settings/maya.json | 2 +- .../schemas/schema_global_publish.json | 4 ++ .../schemas/schema_global_tools.json | 57 +++++++++++++++++++ 4 files changed, 65 insertions(+), 1 deletion(-) diff --git a/openpype/settings/defaults/project_settings/global.json b/openpype/settings/defaults/project_settings/global.json index 0ff9363ba7..3e00cd725e 100644 --- a/openpype/settings/defaults/project_settings/global.json +++ b/openpype/settings/defaults/project_settings/global.json @@ -414,6 +414,9 @@ "filter_families": [] } ] + }, + "publish": { + "template_name_profiles": [] } }, "project_folder_structure": "{\"__project_root__\": {\"prod\": {}, \"resources\": {\"footage\": {\"plates\": {}, \"offline\": {}}, \"audio\": {}, \"art_dept\": {}}, \"editorial\": {}, \"assets\": {\"characters\": {}, \"locations\": {}}, \"shots\": {}}}", diff --git a/openpype/settings/defaults/project_settings/maya.json b/openpype/settings/defaults/project_settings/maya.json index 28f6d23e4d..38063bc2c1 100644 --- a/openpype/settings/defaults/project_settings/maya.json +++ b/openpype/settings/defaults/project_settings/maya.json @@ -980,4 +980,4 @@ "ValidateNoAnimation": false } } -} +} \ No newline at end of file diff --git a/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_publish.json b/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_publish.json index e1aa230b49..c24c88d04a 100644 --- a/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_publish.json +++ b/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_publish.json @@ -642,6 +642,10 @@ ] } }, + { + "type": "label", + "label": "NOTE: Publish template profiles settings were moved to Tools/Publish/Template name profiles. Please move values there." + }, { "type": "list", "key": "template_name_profiles", diff --git a/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_tools.json b/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_tools.json index f8c9482e5f..7dc44c2842 100644 --- a/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_tools.json +++ b/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_tools.json @@ -284,6 +284,63 @@ } } ] + }, + { + "type": "dict", + "key": "publish", + "label": "Publish", + "children": [ + { + "type": "label", + "label": "NOTE: For backwards compatibility can be value empty and in that case are used values from IntegrateAssetNew. This will change in future so please move all values here as soon as possible." + }, + { + "type": "list", + "key": "template_name_profiles", + "label": "Template name profiles", + "use_label_wrap": true, + "object_type": { + "type": "dict", + "children": [ + { + "type": "label", + "label": "" + }, + { + "key": "families", + "label": "Families", + "type": "list", + "object_type": "text" + }, + { + "type": "hosts-enum", + "key": "hosts", + "label": "Hosts", + "multiselection": true + }, + { + "key": "task_types", + "label": "Task types", + "type": "task-types-enum" + }, + { + "key": "tasks", + "label": "Task names", + "type": "list", + "object_type": "text" + }, + { + "type": "separator" + }, + { + "type": "text", + "key": "template_name", + "label": "Template name" + } + ] + } + } + ] } ] } From 2b6c4659237259b6c691dd2b5dc1db927b47fcd3 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 29 Aug 2022 18:03:30 +0200 Subject: [PATCH 024/122] added helper functions to get template name --- openpype/pipeline/publish/__init__.py | 4 ++ openpype/pipeline/publish/contants.py | 1 + openpype/pipeline/publish/lib.py | 97 ++++++++++++++++++++++++++- 3 files changed, 100 insertions(+), 2 deletions(-) create mode 100644 openpype/pipeline/publish/contants.py diff --git a/openpype/pipeline/publish/__init__.py b/openpype/pipeline/publish/__init__.py index aa7fe0bdbf..a2aa61c4d5 100644 --- a/openpype/pipeline/publish/__init__.py +++ b/openpype/pipeline/publish/__init__.py @@ -10,6 +10,8 @@ from .publish_plugins import ( ) from .lib import ( + get_publish_template_name, + DiscoverResult, publish_plugins_discover, load_help_content_from_plugin, @@ -33,6 +35,8 @@ __all__ = ( "OpenPypePyblishPluginMixin", "OptionalPyblishPluginMixin", + "get_publish_template_name", + "DiscoverResult", "publish_plugins_discover", "load_help_content_from_plugin", diff --git a/openpype/pipeline/publish/contants.py b/openpype/pipeline/publish/contants.py new file mode 100644 index 0000000000..958675ecc1 --- /dev/null +++ b/openpype/pipeline/publish/contants.py @@ -0,0 +1 @@ +DEFAULT_PUBLISH_TEMPLATE = "publish" diff --git a/openpype/pipeline/publish/lib.py b/openpype/pipeline/publish/lib.py index 9060a0bf4b..7c3ea22c06 100644 --- a/openpype/pipeline/publish/lib.py +++ b/openpype/pipeline/publish/lib.py @@ -8,8 +8,101 @@ import six import pyblish.plugin import pyblish.api -from openpype.lib import Logger -from openpype.settings import get_project_settings, get_system_settings +from openpype.lib import Logger, filter_profiles +from openpype.settings import ( + get_project_settings, + get_system_settings, +) + +from .contants import DEFAULT_PUBLISH_TEMPLATE + + +def get_template_name_profiles(project_name=None, project_settings=None): + """Receive profiles for publish template keys. + + At least one of arguments must be passed. + + Args: + project_name (str): Name of project where to look for templates. + project_settings(Dic[str, Any]): Prepared project settings. + + Returns: + List[Dict[str, Any]]: Publish template profiles. + """ + + if not project_name and not project_settings: + raise ValueError(( + "Both project name and project settings are missing." + " At least one must be entered." + )) + + if not project_settings: + project_settings = get_project_settings(project_name) + + profiles = ( + project_settings + ["global"] + ["tools"] + ["publish"] + ["template_name_profiles"] + ) + if profiles: + return profiles + + # Use legacy approach for cases new settings are not filled yet for the + # project + return ( + project_settings + ["global"] + ["publish"] + ["IntegrateAssetNew"] + ["template_name_profiles"] + ) + + +def get_publish_template_name( + project_name, + host_name, + family, + task_name, + task_type, + project_settings=None, + logger=None +): + """Get template name which should be used for passed context. + + Publish templates are filtered by host name, family, task name and + task type. + + Default template which is used at if profiles are not available or profile + has empty value is defined by 'DEFAULT_PUBLISH_TEMPLATE' constant. + + Args: + project_name (str): Name of project where to look for settings. + host_name (str): Name of host integration. + family (str): Family for which should be found template. + task_name (str): Task name on which is intance working. + task_type (str): Task type on which is intance working. + project_setting (Dict[str, Any]): Prepared project settings. + logger (logging.Logger): Custom logger used for 'filter_profiles' + function. + + Returns: + str: Template name which should be used for integration. + """ + + template = None + filter_criteria = { + "hosts": host_name, + "families": family, + "tasks": task_name, + "task_types": task_type, + } + profiles = get_template_name_profiles(project_name, project_settings) + profile = filter_profiles(profiles, filter_criteria, logger=logger) + if profile: + template = profile["template_name"] + return template or DEFAULT_PUBLISH_TEMPLATE class DiscoverResult: From 96138a0b73ba6a3f9757283853da9cd1aa85c023 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 29 Aug 2022 18:22:49 +0200 Subject: [PATCH 025/122] use new functions in integrators --- openpype/plugins/publish/integrate.py | 61 ++++++-------------- openpype/plugins/publish/integrate_legacy.py | 21 +++---- 2 files changed, 27 insertions(+), 55 deletions(-) diff --git a/openpype/plugins/publish/integrate.py b/openpype/plugins/publish/integrate.py index f99c718f8a..56d2621015 100644 --- a/openpype/plugins/publish/integrate.py +++ b/openpype/plugins/publish/integrate.py @@ -26,7 +26,10 @@ from openpype.lib import source_hash from openpype.lib.profiles_filtering import filter_profiles from openpype.lib.file_transaction import FileTransaction from openpype.pipeline import legacy_io -from openpype.pipeline.publish import KnownPublishError +from openpype.pipeline.publish import ( + KnownPublishError, + get_publish_template_name, +) log = logging.getLogger(__name__) @@ -792,52 +795,26 @@ class IntegrateAsset(pyblish.api.InstancePlugin): def get_template_name(self, instance): """Return anatomy template name to use for integration""" - # Define publish template name from profiles - filter_criteria = self.get_profile_filter_criteria(instance) - template_name_profiles = self._get_template_name_profiles(instance) - profile = filter_profiles( - template_name_profiles, - filter_criteria, - logger=self.log - ) - - if profile: - return profile["template_name"] - return self.default_template_name - - def _get_template_name_profiles(self, instance): - """Receive profiles for publish template keys. - - Reuse template name profiles from legacy integrator. Goal is to move - the profile settings out of plugin settings but until that happens we - want to be able set it at one place and don't break backwards - compatibility (more then once). - """ - - return ( - instance.context.data["project_settings"] - ["global"] - ["publish"] - ["IntegrateAssetNew"] - ["template_name_profiles"] - ) - - def get_profile_filter_criteria(self, instance): - """Return filter criteria for `filter_profiles`""" # Anatomy data is pre-filled by Collectors - anatomy_data = instance.data["anatomyData"] + + project_name = legacy_io.active_project() # Task can be optional in anatomy data - task = anatomy_data.get("task", {}) + host_name = instance.context.data["hostName"] + anatomy_data = instance.data["anatomyData"] + family = anatomy_data["family"] + task_info = anatomy_data.get("task") or {} - # Return filter criteria - return { - "families": anatomy_data["family"], - "tasks": task.get("name"), - "task_types": task.get("type"), - "hosts": instance.context.data["hostName"], - } + return get_publish_template_name( + project_name, + host_name, + family, + task_name=task_info.get("name"), + task_type=task_info.get("type"), + project_settings=instance.context.data["project_settings"], + logger=self.log + ) def get_rootless_path(self, anatomy, path): """Returns, if possible, path without absolute portion from root diff --git a/openpype/plugins/publish/integrate_legacy.py b/openpype/plugins/publish/integrate_legacy.py index b90b61f587..fedaae794a 100644 --- a/openpype/plugins/publish/integrate_legacy.py +++ b/openpype/plugins/publish/integrate_legacy.py @@ -33,6 +33,7 @@ from openpype.lib import ( TemplateUnsolved ) from openpype.pipeline import legacy_io +from openpype.pipeline.publish import get_publish_template_name # this is needed until speedcopy for linux is fixed if sys.platform == "win32": @@ -388,22 +389,16 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): family = self.main_family_from_instance(instance) - key_values = { - "families": family, - "tasks": task_name, - "hosts": instance.context.data["hostName"], - "task_types": task_type - } - profile = filter_profiles( - self.template_name_profiles, - key_values, + template_name = get_publish_template_name( + project_name, + instance.context.data["hostName"], + family, + task_name=task_info.get("name"), + task_type=task_info.get("type"), + project_settings=instance.context.data["project_settings"], logger=self.log ) - template_name = "publish" - if profile: - template_name = profile["template_name"] - published_representations = {} for idx, repre in enumerate(repres): published_files = [] From c7108ac7fbad46fe2aafe669498cb3755d9c7730 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 29 Aug 2022 18:23:37 +0200 Subject: [PATCH 026/122] modified imports in integrators --- openpype/plugins/publish/integrate.py | 6 +++--- openpype/plugins/publish/integrate_legacy.py | 11 ++++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/openpype/plugins/publish/integrate.py b/openpype/plugins/publish/integrate.py index 56d2621015..8b60ea3b51 100644 --- a/openpype/plugins/publish/integrate.py +++ b/openpype/plugins/publish/integrate.py @@ -5,6 +5,9 @@ import copy import clique import six +from bson.objectid import ObjectId +import pyblish.api + from openpype.client.operations import ( OperationsSession, new_subset_document, @@ -14,8 +17,6 @@ from openpype.client.operations import ( prepare_version_update_data, prepare_representation_update_data, ) -from bson.objectid import ObjectId -import pyblish.api from openpype.client import ( get_representations, @@ -23,7 +24,6 @@ from openpype.client import ( get_version_by_name, ) from openpype.lib import source_hash -from openpype.lib.profiles_filtering import filter_profiles from openpype.lib.file_transaction import FileTransaction from openpype.pipeline import legacy_io from openpype.pipeline.publish import ( diff --git a/openpype/plugins/publish/integrate_legacy.py b/openpype/plugins/publish/integrate_legacy.py index fedaae794a..0e157c9d1f 100644 --- a/openpype/plugins/publish/integrate_legacy.py +++ b/openpype/plugins/publish/integrate_legacy.py @@ -15,7 +15,6 @@ from bson.objectid import ObjectId from pymongo import DeleteOne, InsertOne import pyblish.api -import openpype.api from openpype.client import ( get_asset_by_name, get_subset_by_id, @@ -25,12 +24,14 @@ from openpype.client import ( get_representations, get_archived_representations, ) -from openpype.lib.profiles_filtering import filter_profiles from openpype.lib import ( prepare_template_data, create_hard_link, StringTemplate, - TemplateUnsolved + TemplateUnsolved, + source_hash, + filter_profiles, + get_local_site_id, ) from openpype.pipeline import legacy_io from openpype.pipeline.publish import get_publish_template_name @@ -1053,7 +1054,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): for _src, dest in resources: path = self.get_rootless_path(anatomy, dest) dest = self.get_dest_temp_url(dest) - file_hash = openpype.api.source_hash(dest) + file_hash = source_hash(dest) if self.TMP_FILE_EXT and \ ',{}'.format(self.TMP_FILE_EXT) in file_hash: file_hash = file_hash.replace(',{}'.format(self.TMP_FILE_EXT), @@ -1163,7 +1164,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): def _get_sites(self, sync_project_presets): """Returns tuple (local_site, remote_site)""" - local_site_id = openpype.api.get_local_site_id() + local_site_id = get_local_site_id() local_site = sync_project_presets["config"]. \ get("active_site", "studio").strip() From c76a1a1dbbe8e705b06ebf02f37237cf7dda98fd Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 29 Aug 2022 18:52:43 +0200 Subject: [PATCH 027/122] added settings for hero templates and changed 'tasks' to 'task_names' --- .../defaults/project_settings/global.json | 3 +- .../schemas/schema_global_tools.json | 47 +++++++++++++++++-- 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/openpype/settings/defaults/project_settings/global.json b/openpype/settings/defaults/project_settings/global.json index 3e00cd725e..8692f95a04 100644 --- a/openpype/settings/defaults/project_settings/global.json +++ b/openpype/settings/defaults/project_settings/global.json @@ -416,7 +416,8 @@ ] }, "publish": { - "template_name_profiles": [] + "template_name_profiles": [], + "hero_template_name_profiles": [] } }, "project_folder_structure": "{\"__project_root__\": {\"prod\": {}, \"resources\": {\"footage\": {\"plates\": {}, \"offline\": {}}, \"audio\": {}, \"art_dept\": {}}, \"editorial\": {}, \"assets\": {\"characters\": {}, \"locations\": {}}, \"shots\": {}}}", diff --git a/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_tools.json b/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_tools.json index 7dc44c2842..c919cd73c5 100644 --- a/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_tools.json +++ b/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_tools.json @@ -303,9 +303,47 @@ "type": "dict", "children": [ { - "type": "label", - "label": "" + "key": "families", + "label": "Families", + "type": "list", + "object_type": "text" }, + { + "type": "hosts-enum", + "key": "hosts", + "label": "Hosts", + "multiselection": true + }, + { + "key": "task_types", + "label": "Task types", + "type": "task-types-enum" + }, + { + "key": "task_names", + "label": "Task names", + "type": "list", + "object_type": "text" + }, + { + "type": "separator" + }, + { + "type": "text", + "key": "template_name", + "label": "Template name" + } + ] + } + }, + { + "type": "list", + "key": "hero_template_name_profiles", + "label": "Hero template name profiles", + "use_label_wrap": true, + "object_type": { + "type": "dict", + "children": [ { "key": "families", "label": "Families", @@ -324,7 +362,7 @@ "type": "task-types-enum" }, { - "key": "tasks", + "key": "task_names", "label": "Task names", "type": "list", "object_type": "text" @@ -335,7 +373,8 @@ { "type": "text", "key": "template_name", - "label": "Template name" + "label": "Template name", + "tooltip": "Name of template from Anatomy templates" } ] } From 9d4416719b4a99d50b0d411b5548a8afa8072240 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 29 Aug 2022 18:53:33 +0200 Subject: [PATCH 028/122] convert legacy to new settings by replacing 'tasks' with 'task_names' --- openpype/pipeline/publish/lib.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/openpype/pipeline/publish/lib.py b/openpype/pipeline/publish/lib.py index 7c3ea22c06..03dfbadfcc 100644 --- a/openpype/pipeline/publish/lib.py +++ b/openpype/pipeline/publish/lib.py @@ -2,6 +2,7 @@ import os import sys import types import inspect +import copy import xml.etree.ElementTree import six @@ -47,17 +48,23 @@ def get_template_name_profiles(project_name=None, project_settings=None): ["template_name_profiles"] ) if profiles: - return profiles + return copy.deepcopy(profiles) # Use legacy approach for cases new settings are not filled yet for the # project - return ( + legacy_profiles = ( project_settings ["global"] ["publish"] ["IntegrateAssetNew"] ["template_name_profiles"] ) + # Replace "tasks" key with "task_names" + profiles = [] + for profile in copy.deepcopy(legacy_profiles): + profile["task_names"] = profile.pop("tasks", []) + profiles.append(profile) + return profiles def get_publish_template_name( @@ -95,7 +102,7 @@ def get_publish_template_name( filter_criteria = { "hosts": host_name, "families": family, - "tasks": task_name, + "task_names": task_name, "task_types": task_type, } profiles = get_template_name_profiles(project_name, project_settings) From 63f5b5f2ab40a94c7496b8f08fa19204a5687b5a Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 29 Aug 2022 18:53:49 +0200 Subject: [PATCH 029/122] added ability to get hero version template name --- openpype/pipeline/publish/contants.py | 1 + openpype/pipeline/publish/lib.py | 64 +++++++++++++++++++++++++-- 2 files changed, 62 insertions(+), 3 deletions(-) diff --git a/openpype/pipeline/publish/contants.py b/openpype/pipeline/publish/contants.py index 958675ecc1..169eca2e5c 100644 --- a/openpype/pipeline/publish/contants.py +++ b/openpype/pipeline/publish/contants.py @@ -1 +1,2 @@ DEFAULT_PUBLISH_TEMPLATE = "publish" +DEFAULT_HERO_PUBLISH_TEMPLATE = "hero" diff --git a/openpype/pipeline/publish/lib.py b/openpype/pipeline/publish/lib.py index 03dfbadfcc..85a64da721 100644 --- a/openpype/pipeline/publish/lib.py +++ b/openpype/pipeline/publish/lib.py @@ -15,7 +15,10 @@ from openpype.settings import ( get_system_settings, ) -from .contants import DEFAULT_PUBLISH_TEMPLATE +from .contants import ( + DEFAULT_PUBLISH_TEMPLATE, + DEFAULT_HERO_PUBLISH_TEMPLATE, +) def get_template_name_profiles(project_name=None, project_settings=None): @@ -67,6 +70,49 @@ def get_template_name_profiles(project_name=None, project_settings=None): return profiles +def get_hero_template_name_profiles(project_name=None, project_settings=None): + """Receive profiles for hero publish template keys. + + At least one of arguments must be passed. + + Args: + project_name (str): Name of project where to look for templates. + project_settings(Dic[str, Any]): Prepared project settings. + + Returns: + List[Dict[str, Any]]: Publish template profiles. + """ + + if not project_name and not project_settings: + raise ValueError(( + "Both project name and project settings are missing." + " At least one must be entered." + )) + + if not project_settings: + project_settings = get_project_settings(project_name) + + profiles = ( + project_settings + ["global"] + ["tools"] + ["publish"] + ["hero_template_name_profiles"] + ) + if profiles: + return copy.deepcopy(profiles) + + # Use legacy approach for cases new settings are not filled yet for the + # project + return copy.deepcopy( + project_settings + ["global"] + ["publish"] + ["IntegrateHeroVersion"] + ["template_name_profiles"] + ) + + def get_publish_template_name( project_name, host_name, @@ -74,6 +120,7 @@ def get_publish_template_name( task_name, task_type, project_settings=None, + hero=False, logger=None ): """Get template name which should be used for passed context. @@ -105,11 +152,22 @@ def get_publish_template_name( "task_names": task_name, "task_types": task_type, } - profiles = get_template_name_profiles(project_name, project_settings) + if hero: + default_template = DEFAULT_HERO_PUBLISH_TEMPLATE + profiles = get_hero_template_name_profiles( + project_name, project_settings + ) + + else: + profiles = get_template_name_profiles( + project_name, project_settings + ) + default_template = DEFAULT_PUBLISH_TEMPLATE + profile = filter_profiles(profiles, filter_criteria, logger=logger) if profile: template = profile["template_name"] - return template or DEFAULT_PUBLISH_TEMPLATE + return template or default_template class DiscoverResult: From 1698aefcfbc887ba6f29fc59dbdfbc2595d5c6a8 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 29 Aug 2022 18:57:00 +0200 Subject: [PATCH 030/122] use 'get_publish_template_name' in hero integration --- .../plugins/publish/integrate_hero_version.py | 41 ++++++++----------- 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/openpype/plugins/publish/integrate_hero_version.py b/openpype/plugins/publish/integrate_hero_version.py index 7d698ff98d..2938c61f8e 100644 --- a/openpype/plugins/publish/integrate_hero_version.py +++ b/openpype/plugins/publish/integrate_hero_version.py @@ -14,14 +14,12 @@ from openpype.client import ( get_archived_representations, get_representations, ) -from openpype.lib import ( - create_hard_link, - filter_profiles -) +from openpype.lib import create_hard_link from openpype.pipeline import ( schema, legacy_io, ) +from openpype.pipeline.publish import get_publish_template_name class IntegrateHeroVersion(pyblish.api.InstancePlugin): @@ -68,10 +66,11 @@ class IntegrateHeroVersion(pyblish.api.InstancePlugin): ) return - template_key = self._get_template_key(instance) - anatomy = instance.context.data["anatomy"] project_name = anatomy.project_name + + template_key = self._get_template_key(project_name, instance) + if template_key not in anatomy.templates: self.log.warning(( "!!! Anatomy of project \"{}\" does not have set" @@ -527,30 +526,24 @@ class IntegrateHeroVersion(pyblish.api.InstancePlugin): return publish_folder - def _get_template_key(self, instance): + def _get_template_key(self, project_name, instance): anatomy_data = instance.data["anatomyData"] - task_data = anatomy_data.get("task") or {} - task_name = task_data.get("name") - task_type = task_data.get("type") + task_info = anatomy_data.get("task") or {} host_name = instance.context.data["hostName"] + # TODO raise error if Hero not set? family = self.main_family_from_instance(instance) - key_values = { - "families": family, - "task_names": task_name, - "task_types": task_type, - "hosts": host_name - } - profile = filter_profiles( - self.template_name_profiles, - key_values, + + return get_publish_template_name( + project_name, + host_name, + family, + task_info.get("name"), + task_info.get("type"), + project_settings=instance.context.data["project_settings"], + hero=True, logger=self.log ) - if profile: - template_name = profile["template_name"] - else: - template_name = self._default_template_name - return template_name def main_family_from_instance(self, instance): """Returns main family of entered instance.""" From 9b7384e1ae96b0f348911e9e163a23857dd2ca7f Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 29 Aug 2022 18:57:27 +0200 Subject: [PATCH 031/122] removed unused attribute --- openpype/plugins/publish/integrate_legacy.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openpype/plugins/publish/integrate_legacy.py b/openpype/plugins/publish/integrate_legacy.py index 0e157c9d1f..bbf30c9ab7 100644 --- a/openpype/plugins/publish/integrate_legacy.py +++ b/openpype/plugins/publish/integrate_legacy.py @@ -140,7 +140,6 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): integrated_file_sizes = {} # Attributes set by settings - template_name_profiles = None subset_grouping_profiles = None def process(self, instance): From c6a6e3b21a4aaa6c98450de918c78907fed91f5e Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 29 Aug 2022 19:07:34 +0200 Subject: [PATCH 032/122] added warning for access to legacy settings --- openpype/pipeline/publish/lib.py | 35 +++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/openpype/pipeline/publish/lib.py b/openpype/pipeline/publish/lib.py index 85a64da721..29c745ed15 100644 --- a/openpype/pipeline/publish/lib.py +++ b/openpype/pipeline/publish/lib.py @@ -21,7 +21,9 @@ from .contants import ( ) -def get_template_name_profiles(project_name=None, project_settings=None): +def get_template_name_profiles( + project_name, project_settings=None, logger=None +): """Receive profiles for publish template keys. At least one of arguments must be passed. @@ -62,6 +64,16 @@ def get_template_name_profiles(project_name=None, project_settings=None): ["IntegrateAssetNew"] ["template_name_profiles"] ) + if legacy_profiles: + if not logger: + logger = Logger.get_logger("get_template_name_profiles") + + logger.warning(( + "Project \"{}\" is using legacy access to publish template." + " It is recommended to move settings to new location" + " 'project_settings/global/tools/publish/template_name_profiles'." + ).format(project_name)) + # Replace "tasks" key with "task_names" profiles = [] for profile in copy.deepcopy(legacy_profiles): @@ -70,7 +82,9 @@ def get_template_name_profiles(project_name=None, project_settings=None): return profiles -def get_hero_template_name_profiles(project_name=None, project_settings=None): +def get_hero_template_name_profiles( + project_name, project_settings=None, logger=None +): """Receive profiles for hero publish template keys. At least one of arguments must be passed. @@ -104,13 +118,24 @@ def get_hero_template_name_profiles(project_name=None, project_settings=None): # Use legacy approach for cases new settings are not filled yet for the # project - return copy.deepcopy( + legacy_profiles = copy.deepcopy( project_settings ["global"] ["publish"] ["IntegrateHeroVersion"] ["template_name_profiles"] ) + if legacy_profiles: + if not logger: + logger = Logger.get_logger("get_hero_template_name_profiles") + + logger.warning(( + "Project \"{}\" is using legacy access to hero publish template." + " It is recommended to move settings to new location" + " 'project_settings/global/tools/publish/" + "hero_template_name_profiles'." + ).format(project_name)) + return legacy_profiles def get_publish_template_name( @@ -155,12 +180,12 @@ def get_publish_template_name( if hero: default_template = DEFAULT_HERO_PUBLISH_TEMPLATE profiles = get_hero_template_name_profiles( - project_name, project_settings + project_name, project_settings, logger ) else: profiles = get_template_name_profiles( - project_name, project_settings + project_name, project_settings, logger ) default_template = DEFAULT_PUBLISH_TEMPLATE From dc4c32b6fc6aaeaedb6bb9d76c7b72d9f5c45c45 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 29 Aug 2022 19:14:58 +0200 Subject: [PATCH 033/122] Fix representation data for workfile --- openpype/modules/deadline/abstract_submit_deadline.py | 6 +++--- .../deadline/plugins/publish/submit_maya_deadline.py | 3 --- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py index 577378335e..d198542370 100644 --- a/openpype/modules/deadline/abstract_submit_deadline.py +++ b/openpype/modules/deadline/abstract_submit_deadline.py @@ -554,9 +554,9 @@ class AbstractSubmitDeadline(pyblish.api.InstancePlugin): "Workfile (scene) must be published along") # determine published path from Anatomy. template_data = i.data.get("anatomyData") - rep = i.data.get("representations")[0].get("ext") - template_data["representation"] = rep - template_data["ext"] = rep + rep = i.data.get("representations")[0] + template_data["representation"] = rep.get("name") + template_data["ext"] = rep.get("ext") template_data["comment"] = None anatomy_filled = anatomy.format(template_data) template_filled = anatomy_filled["publish"]["path"] diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 5a7d0b98c6..68e8eaaa73 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -258,10 +258,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): ) self._patch_workfile(filepath, patches) - # todo: on self.use_published originally use template_data["representation"] using .get("name") instead of .get("ext") # todo: on self.use_published replace path for publishRenderMetadataFolder - # rep = i.data.get("representations")[0].get("name") - # if instance.data.get("publishRenderMetadataFolder"): # instance.data["publishRenderMetadataFolder"] = \ # instance.data["publishRenderMetadataFolder"].replace( From 67b8664be07fbb8a0061b7c8a62bf4073ef79307 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 29 Aug 2022 19:15:20 +0200 Subject: [PATCH 034/122] Remove comment for patched file code refactor since it's already implemented --- .../modules/deadline/plugins/publish/submit_maya_deadline.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 68e8eaaa73..07ed237c94 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -266,10 +266,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): # self.log.info("Scene name was switched {} -> {}".format( # orig_scene, new_scene # )) - # # patch workfile is needed - # if filepath not in patched_files: - # patched_file = self._patch_workfile(filepath, patches) - # patched_files.append(patched_file) # Gather needed data ------------------------------------------------ workspace = context.data["workspaceDir"] From 2da8f036dee501be62da07d07fa9efafc9e8839f Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 29 Aug 2022 19:19:37 +0200 Subject: [PATCH 035/122] Refactor logic for less indentation --- .../deadline/abstract_submit_deadline.py | 103 +++++++++--------- 1 file changed, 52 insertions(+), 51 deletions(-) diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py index d198542370..55e16d8d21 100644 --- a/openpype/modules/deadline/abstract_submit_deadline.py +++ b/openpype/modules/deadline/abstract_submit_deadline.py @@ -546,65 +546,66 @@ class AbstractSubmitDeadline(pyblish.api.InstancePlugin): anatomy = self._instance.context.data['anatomy'] file_path = None for i in self._instance.context: - if "workfile" in i.data["families"] \ - or i.data["family"] == "workfile": - # test if there is instance of workfile waiting - # to be published. - assert i.data["publish"] is True, ( - "Workfile (scene) must be published along") - # determine published path from Anatomy. - template_data = i.data.get("anatomyData") - rep = i.data.get("representations")[0] - template_data["representation"] = rep.get("name") - template_data["ext"] = rep.get("ext") - template_data["comment"] = None - anatomy_filled = anatomy.format(template_data) - template_filled = anatomy_filled["publish"]["path"] - file_path = os.path.normpath(template_filled) - self.log.info("Using published scene for render {}".format( - file_path)) + is_workfile = + if not is_workfile: + continue - if not os.path.exists(file_path): - self.log.error("published scene does not exist!") - raise + # test if there is instance of workfile waiting + # to be published. + assert i.data["publish"] is True, ( + "Workfile (scene) must be published along") + # determine published path from Anatomy. + template_data = i.data.get("anatomyData") + rep = i.data.get("representations")[0] + template_data["representation"] = rep.get("name") + template_data["ext"] = rep.get("ext") + template_data["comment"] = None + anatomy_filled = anatomy.format(template_data) + template_filled = anatomy_filled["publish"]["path"] + file_path = os.path.normpath(template_filled) - if not replace_in_path: - return file_path + self.log.info("Using published scene for render {}".format( + file_path)) - # now we need to switch scene in expected files - # because token will now point to published - # scene file and that might differ from current one - new_scene = os.path.splitext( - os.path.basename(file_path))[0] - orig_scene = os.path.splitext( - os.path.basename( - self._instance.context.data["currentFile"]))[0] - exp = self._instance.data.get("expectedFiles") + if not os.path.exists(file_path): + self.log.error("published scene does not exist!") + raise - if isinstance(exp[0], dict): - # we have aovs and we need to iterate over them - new_exp = {} - for aov, files in exp[0].items(): - replaced_files = [] - for f in files: - replaced_files.append( - str(f).replace(orig_scene, new_scene) - ) - new_exp[aov] = replaced_files - # [] might be too much here, TODO - self._instance.data["expectedFiles"] = [new_exp] - else: - new_exp = [] - for f in exp: - new_exp.append( + if not replace_in_path: + return file_path + + # now we need to switch scene in expected files + # because token will now point to published + # scene file and that might differ from current one + new_scene = os.path.splitext(os.path.basename(file_path))[0] + orig_scene = os.path.splitext(os.path.basename( + self._instance.context.data["currentFile"]))[0] + exp = self._instance.data.get("expectedFiles") + + if isinstance(exp[0], dict): + # we have aovs and we need to iterate over them + new_exp = {} + for aov, files in exp[0].items(): + replaced_files = [] + for f in files: + replaced_files.append( str(f).replace(orig_scene, new_scene) ) - self._instance.data["expectedFiles"] = new_exp + new_exp[aov] = replaced_files + # [] might be too much here, TODO + self._instance.data["expectedFiles"] = [new_exp] + else: + new_exp = [] + for f in exp: + new_exp.append( + str(f).replace(orig_scene, new_scene) + ) + self._instance.data["expectedFiles"] = new_exp - self.log.info("Scene name was switched {} -> {}".format( - orig_scene, new_scene - )) + self.log.info("Scene name was switched {} -> {}".format( + orig_scene, new_scene + )) return file_path From 21a319b10c46dec6efc2aaf6fe6ac1fe09bfc512 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 29 Aug 2022 19:20:33 +0200 Subject: [PATCH 036/122] added 'deprecated' to integrator labels and added new location for hero templates as note --- .../projects_schema/schemas/schema_global_publish.json | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_publish.json b/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_publish.json index c24c88d04a..2cb0cebf95 100644 --- a/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_publish.json +++ b/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_publish.json @@ -649,7 +649,7 @@ { "type": "list", "key": "template_name_profiles", - "label": "Template name profiles", + "label": "Template name profiles (DEPRECATED)", "use_label_wrap": true, "object_type": { "type": "dict", @@ -754,10 +754,14 @@ "type": "list", "object_type": "text" }, + { + "type": "label", + "label": "NOTE: Hero publish template profiles settings were moved to Tools/Publish/Hero template name profiles. Please move values there." + }, { "type": "list", "key": "template_name_profiles", - "label": "Template name profiles", + "label": "Template name profiles (DEPRECATED)", "use_label_wrap": true, "object_type": { "type": "dict", From e81e3a7a1021db4e442aa3147ed03ccf8d92d8c6 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 29 Aug 2022 19:22:06 +0200 Subject: [PATCH 037/122] Fix missing line --- openpype/modules/deadline/abstract_submit_deadline.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py index 55e16d8d21..86eebc0d35 100644 --- a/openpype/modules/deadline/abstract_submit_deadline.py +++ b/openpype/modules/deadline/abstract_submit_deadline.py @@ -547,7 +547,10 @@ class AbstractSubmitDeadline(pyblish.api.InstancePlugin): file_path = None for i in self._instance.context: - is_workfile = + is_workfile = ( + "workfile" in i.data.get("families", []) or + i.data["family"] == "workfile" + ) if not is_workfile: continue From 2933b37ef7711aac4e04284120b6c9b0ce2c9612 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 29 Aug 2022 19:34:27 +0200 Subject: [PATCH 038/122] Refactor code for readability --- .../deadline/abstract_submit_deadline.py | 124 ++++++++++-------- 1 file changed, 68 insertions(+), 56 deletions(-) diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py index 86eebc0d35..46baa9ee57 100644 --- a/openpype/modules/deadline/abstract_submit_deadline.py +++ b/openpype/modules/deadline/abstract_submit_deadline.py @@ -543,72 +543,84 @@ class AbstractSubmitDeadline(pyblish.api.InstancePlugin): published. """ - anatomy = self._instance.context.data['anatomy'] - file_path = None - for i in self._instance.context: - is_workfile = ( - "workfile" in i.data.get("families", []) or - i.data["family"] == "workfile" - ) - if not is_workfile: - continue + def _get_workfile_instance(context): + """Find workfile instance in context""" + for i in context: - # test if there is instance of workfile waiting - # to be published. - assert i.data["publish"] is True, ( - "Workfile (scene) must be published along") - # determine published path from Anatomy. - template_data = i.data.get("anatomyData") - rep = i.data.get("representations")[0] - template_data["representation"] = rep.get("name") - template_data["ext"] = rep.get("ext") - template_data["comment"] = None - anatomy_filled = anatomy.format(template_data) - template_filled = anatomy_filled["publish"]["path"] - file_path = os.path.normpath(template_filled) + is_workfile = ( + "workfile" in i.data.get("families", []) or + i.data["family"] == "workfile" + ) + if not is_workfile: + continue - self.log.info("Using published scene for render {}".format( - file_path)) + # test if there is instance of workfile waiting + # to be published. + assert i.data["publish"] is True, ( + "Workfile (scene) must be published along") - if not os.path.exists(file_path): - self.log.error("published scene does not exist!") - raise + return i - if not replace_in_path: - return file_path + instance = self._instance + workfile_instance = _get_workfile_instance(instance.context) + if not workfile_instance: + return - # now we need to switch scene in expected files - # because token will now point to published - # scene file and that might differ from current one - new_scene = os.path.splitext(os.path.basename(file_path))[0] - orig_scene = os.path.splitext(os.path.basename( - self._instance.context.data["currentFile"]))[0] - exp = self._instance.data.get("expectedFiles") + # determine published path from Anatomy. + template_data = workfile_instance.data.get("anatomyData") + rep = workfile_instance.data.get("representations")[0] + template_data["representation"] = rep.get("name") + template_data["ext"] = rep.get("ext") + template_data["comment"] = None - if isinstance(exp[0], dict): - # we have aovs and we need to iterate over them - new_exp = {} - for aov, files in exp[0].items(): - replaced_files = [] - for f in files: - replaced_files.append( - str(f).replace(orig_scene, new_scene) - ) - new_exp[aov] = replaced_files - # [] might be too much here, TODO - self._instance.data["expectedFiles"] = [new_exp] - else: - new_exp = [] - for f in exp: - new_exp.append( + anatomy = instance.context.data['anatomy'] + anatomy_filled = anatomy.format(template_data) + template_filled = anatomy_filled["publish"]["path"] + file_path = os.path.normpath(template_filled) + + self.log.info("Using published scene for render {}".format(file_path)) + + if not os.path.exists(file_path): + self.log.error("published scene does not exist!") + raise + + if not replace_in_path: + return file_path + + # now we need to switch scene in expected files + # because token will now point to published + # scene file and that might differ from current one + def _clean_name(path): + return os.path.splitext(os.path.basename(path))[0] + + new_scene = _clean_name(file_path) + orig_scene = _clean_name(instance.context.data["currentFile"]) + expected_files = instance.data.get("expectedFiles") + + if isinstance(expected_files[0], dict): + # we have aovs and we need to iterate over them + new_exp = {} + for aov, files in expected_files[0].items(): + replaced_files = [] + for f in files: + replaced_files.append( str(f).replace(orig_scene, new_scene) ) - self._instance.data["expectedFiles"] = new_exp + new_exp[aov] = replaced_files + # [] might be too much here, TODO + instance.data["expectedFiles"] = [new_exp] + else: + new_exp = [] + for f in expected_files: + new_exp.append( + str(f).replace(orig_scene, new_scene) + ) + instance.data["expectedFiles"] = new_exp - self.log.info("Scene name was switched {} -> {}".format( - orig_scene, new_scene - )) + self.log.info("Scene name was switched {} -> {}".format( + orig_scene, new_scene + )) return file_path From c725ff5b42c2f3f248a6af8f835020c9efb23182 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 29 Aug 2022 19:35:05 +0200 Subject: [PATCH 039/122] Move replacing in `publishRenderMetadataFolder` to abstract base class --- openpype/modules/deadline/abstract_submit_deadline.py | 6 ++++++ .../deadline/plugins/publish/submit_maya_deadline.py | 9 --------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py index 46baa9ee57..f56cf49f6d 100644 --- a/openpype/modules/deadline/abstract_submit_deadline.py +++ b/openpype/modules/deadline/abstract_submit_deadline.py @@ -618,6 +618,12 @@ class AbstractSubmitDeadline(pyblish.api.InstancePlugin): ) instance.data["expectedFiles"] = new_exp + metadata_folder = instance.data.get("publishRenderMetadataFolder") + if metadata_folder: + metadata_folder = metadata_folder.replace(orig_scene, + new_scene) + instance.data["publishRenderMetadataFolder"] = metadata_folder + self.log.info("Scene name was switched {} -> {}".format( orig_scene, new_scene )) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 07ed237c94..26c26a124c 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -258,15 +258,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): ) self._patch_workfile(filepath, patches) - # todo: on self.use_published replace path for publishRenderMetadataFolder - # if instance.data.get("publishRenderMetadataFolder"): - # instance.data["publishRenderMetadataFolder"] = \ - # instance.data["publishRenderMetadataFolder"].replace( - # orig_scene, new_scene) - # self.log.info("Scene name was switched {} -> {}".format( - # orig_scene, new_scene - # )) - # Gather needed data ------------------------------------------------ workspace = context.data["workspaceDir"] default_render_file = instance.context.data.get('project_settings')\ From 23e652a51f41a6c65d6feba6edb0268f85feccb8 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 29 Aug 2022 19:54:36 +0200 Subject: [PATCH 040/122] Patch plug-in payload with settings --- .../maya/plugins/publish/collect_render.py | 1 + .../plugins/publish/submit_maya_deadline.py | 17 +++++++++-------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/collect_render.py b/openpype/hosts/maya/plugins/publish/collect_render.py index ebda5e190d..768a53329f 100644 --- a/openpype/hosts/maya/plugins/publish/collect_render.py +++ b/openpype/hosts/maya/plugins/publish/collect_render.py @@ -293,6 +293,7 @@ class CollectMayaRender(pyblish.api.ContextPlugin): "source": filepath, "expectedFiles": full_exp_files, "publishRenderMetadataFolder": common_publish_meta_path, + "renderProducts": layer_render_products, "resolutionWidth": lib.get_attr_in_layer( "defaultResolution.width", layer=layer_name ), diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 26c26a124c..854a66eaa5 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -215,16 +215,21 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): renderlayer = instance.data['setMembers'] # rs_beauty - self.payload_skeleton["PluginInfo"]["RenderLayer"] = renderlayer - self.payload_skeleton["PluginInfo"]["RenderSetupIncludeLights"] = instance.data.get("renderSetupIncludeLights") # noqa - # Output driver to render plugin_info = DeadlinePluginInfo( SceneFile=context.data["currentFile"], Version=cmds.about(version=True), + RenderLayer=renderlayer, + RenderSetupIncludeLights=instance.data.get("renderSetupIncludeLights") # noqa ) - return attr.asdict(plugin_info) + plugin_payload = attr.asdict(plugin_info) + + # Patching with pluginInfo from settings + for key, value in self.pluginInfo.items(): + plugin_payload[key] = value + + return plugin_payload def process_submission(self): @@ -338,10 +343,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): # Store output dir for unified publisher (filesequence) instance.data["outputDir"] = os.path.dirname(output_filename_0) - # add jobInfo and pluginInfo variables from Settings - payload["JobInfo"].update(self.jobInfo) - payload["PluginInfo"].update(self.pluginInfo) - if instance.data.get("tileRendering"): # Prepare tiles data self._tile_render(instance, payload) From 4abddd027de9d4a1814ddff1b971bb9a99c47008 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 29 Aug 2022 19:55:07 +0200 Subject: [PATCH 041/122] Use collected render products for image prefix --- .../deadline/plugins/publish/submit_maya_deadline.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 854a66eaa5..bb7ae380b6 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -307,7 +307,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): "jobname": jobname, "comment": comment, "output_filename_0": output_filename_0, - "render_variables": render_variables, "renderlayer": renderlayer, "workspace": workspace, "dirname": dirname, @@ -564,6 +563,11 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): renderer = self._instance.data["renderer"] + # Get layer prefix + render_products = self._instance.data["renderProducts"] + layer_metadata = render_products.layer_data + layer_prefix = layer_metadata.filePrefix + # This hack is here because of how Deadline handles Renderman version. # it considers everything with `renderman` set as version older than # Renderman 22, and so if we are using renderman > 21 we need to set @@ -583,7 +587,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): "SceneFile": data["filepath"], # Output directory and filename "OutputFilePath": data["dirname"].replace("\\", "/"), - "OutputFilePrefix": data["render_variables"]["filename_prefix"], # noqa: E501 + "OutputFilePrefix": layer_prefix, # Only render layers are considered renderable in this pipeline "UsingRenderLayers": True, From 6f5fcecfae7ec92bf2df80fb673bfff3e1049231 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 29 Aug 2022 19:55:24 +0200 Subject: [PATCH 042/122] Use existing variable `renderer` --- .../modules/deadline/plugins/publish/submit_maya_deadline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index bb7ae380b6..c7f91905ea 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -573,7 +573,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): # Renderman 22, and so if we are using renderman > 21 we need to set # renderer string on the job to `renderman22`. We will have to change # this when Deadline releases new version handling this. - if self._instance.data["renderer"] == "renderman": + if renderer == "renderman": try: from rfm2.config import cfg # noqa except ImportError: From ae250c4a100dcc4474512913dbc858326ced3c8a Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 29 Aug 2022 19:55:40 +0200 Subject: [PATCH 043/122] Remove unused `comment` key-value --- .../modules/deadline/plugins/publish/submit_maya_deadline.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index c7f91905ea..b2b877ab0e 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -305,7 +305,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): "filename": filename, "filepath": filepath, "jobname": jobname, - "comment": comment, "output_filename_0": output_filename_0, "renderlayer": renderlayer, "workspace": workspace, From 7af7f71edacea21a00c370e4ad1e92b2fe576b66 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 29 Aug 2022 19:58:04 +0200 Subject: [PATCH 044/122] Remove logging of plugin name --- .../modules/deadline/plugins/publish/submit_maya_deadline.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index b2b877ab0e..db796f25a9 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -335,9 +335,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): if export_job: payload["JobInfo"]["JobDependency0"] = export_job - plugin = payload["JobInfo"]["Plugin"] - self.log.info("using render plugin : {}".format(plugin)) - # Store output dir for unified publisher (filesequence) instance.data["outputDir"] = os.path.dirname(output_filename_0) From f91e33c0385762a7a73cca192f4d36716377ee1e Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 29 Aug 2022 23:21:30 +0200 Subject: [PATCH 045/122] More refactoring/cleanup (WIP) --- .../plugins/publish/submit_maya_deadline.py | 304 +++++++----------- 1 file changed, 110 insertions(+), 194 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index db796f25a9..8f12a9518f 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -70,7 +70,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): tile_assembler_plugin = "OpenPypeTileAssembler" priority = 50 tile_priority = 50 - limit_groups = [] + limit = [] # limit groups jobInfo = {} pluginInfo = {} group = "none" @@ -112,23 +112,18 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): job_info.ChunkSize = instance.data.get("chunkSize", 10) job_info.Comment = context.data.get("comment") job_info.Priority = instance.data.get("priority", self.priority) + job_info.FramesPerTask = instance.data.get("framesPerTask", 1) if self.group != "none" and self.group: job_info.Group = self.group - if self.limit_groups: - job_info.LimitGroups = ",".join(self.limit_groups) + if self.limit: + job_info.LimitGroups = ",".join(self.limit) - # Optional, enable double-click to preview rendered - # frames from Deadline Monitor - self.payload_skeleton["JobInfo"]["OutputDirectory0"] = \ - os.path.dirname(output_filename_0).replace("\\", "/") - self.payload_skeleton["JobInfo"]["OutputFilename0"] = \ - output_filename_0.replace("\\", "/") - - # Add options from RenderGlobals------------------------------------- + # Add options from RenderGlobals render_globals = instance.data.get("renderGlobals", {}) - self.payload_skeleton["JobInfo"].update(render_globals) + for key, value in render_globals: + setattr(job_info, key, value) keys = [ "FTRACK_API_KEY", @@ -140,7 +135,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): "AVALON_TASK", "AVALON_APP_NAME", "OPENPYPE_DEV", - "OPENPYPE_LOG_NO_COLORS", "OPENPYPE_VERSION" ] # Add mongo url if it's enabled @@ -150,10 +144,8 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): environment = dict({key: os.environ[key] for key in keys if key in os.environ}, **legacy_io.Session) - # TODO: Taken from old publish class - test whether still needed environment["OPENPYPE_LOG_NO_COLORS"] = "1" - environment["OPENPYPE_MAYA_VERSION"] = cmds.about(v=True) # to recognize job from PYPE for turning Event On/Off environment["OPENPYPE_RENDER_JOB"] = "1" @@ -166,7 +158,10 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): ) # to recognize job from PYPE for turning Event On/Off job_info.EnvironmentKeyValue = "OPENPYPE_RENDER_JOB=1" + job_info.EnvironmentKeyValue = "OPENPYPE_LOG_NO_COLORS=1" + # Optional, enable double-click to preview rendered + # frames from Deadline Monitor for i, filepath in enumerate(instance.data["files"]): dirname = os.path.dirname(filepath) fname = os.path.basename(filepath) @@ -213,14 +208,13 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): instance = self._instance context = instance.context - renderlayer = instance.data['setMembers'] # rs_beauty - - # Output driver to render plugin_info = DeadlinePluginInfo( - SceneFile=context.data["currentFile"], + SceneFile=self.scene_path, Version=cmds.about(version=True), - RenderLayer=renderlayer, - RenderSetupIncludeLights=instance.data.get("renderSetupIncludeLights") # noqa + RenderLayer=instance.data['setMembers'], + RenderSetupIncludeLights=instance.data.get("renderSetupIncludeLights"), # noqa + ProjectPath=context.data["workspaceDir"], + UsingRenderLayers=True, ) plugin_payload = attr.asdict(plugin_info) @@ -236,12 +230,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): instance = self._instance context = instance.context - # Generated by AbstractSubmitDeadline. The `job_info`, `plugin_info` - # and `aux_files` are the skeleton payloads that are the basis for - # all the maya submissions - job_info = self.job_info - plugin_info = self.plugin_info - aux_files = self.aux_files filepath = self.scene_path # publish if `use_publish` else workfile # TODO: Avoid the need for this logic here, needed for submit publish @@ -250,18 +238,9 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): instance.data["outputDir"] = output_dir instance.data["toBeRenderedOn"] = "deadline" - self.limit_groups = self.limit - # Patch workfile (only when use_published is enabled) if self.use_published: - patches = ( - context.data["project_settings"].get( - "deadline", {}).get( - "publish", {}).get( - "MayaSubmitDeadline", {}).get( - "scene_patches", {}) - ) - self._patch_workfile(filepath, patches) + self._patch_workfile() # Gather needed data ------------------------------------------------ workspace = context.data["workspaceDir"] @@ -271,22 +250,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): .get('default_render_image_folder') filename = os.path.basename(filepath) dirname = os.path.join(workspace, default_render_file) - renderlayer = instance.data['setMembers'] # rs_beauty - - # Get the variables depending on the renderer - # TODO: Find replacement logic for `get_renderer_variables` through - # what is collected for the render or is implemented in maya - # api `lib_renderproducts` - render_variables = get_renderer_variables(renderlayer, dirname) - filename_0 = render_variables["filename_0"] - if self.use_published: - new_scene = os.path.splitext(filename)[0] - orig_scene = os.path.splitext( - os.path.basename(context.data["currentFile"]))[0] - filename_0 = render_variables["filename_0"].replace( - orig_scene, new_scene) - - output_filename_0 = filename_0 # this is needed because renderman handles directory and file # prefixes separately @@ -301,16 +264,18 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): pass # Fill in common data to payload ------------------------------------ + # TODO: Replace these with collected data from CollectRender payload_data = { "filename": filename, "filepath": filepath, - "jobname": jobname, "output_filename_0": output_filename_0, "renderlayer": renderlayer, - "workspace": workspace, "dirname": dirname, } + # Store output dir for unified publisher (filesequence) + instance.data["outputDir"] = os.path.dirname(output_filename_0) + # Submit preceding export jobs ------------------------------------- export_job = None assert not all(x in instance.data["families"] @@ -333,17 +298,16 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): # Add export job as dependency -------------------------------------- if export_job: - payload["JobInfo"]["JobDependency0"] = export_job - - # Store output dir for unified publisher (filesequence) - instance.data["outputDir"] = os.path.dirname(output_filename_0) + job_info, _ = payload + job_info.JobDependency = export_job if instance.data.get("tileRendering"): # Prepare tiles data self._tile_render(instance, payload) else: # Submit main render job - self.submit(payload) + job_info, plugin_info = payload + self.submit(self.assemble_payload(job_info, plugin_info)) def _tile_render(self, instance, payload): @@ -546,18 +510,12 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): instance.data["jobBatchName"])) def _get_maya_payload(self, data): - payload = copy.deepcopy(self.payload_skeleton) - if not self.asset_dependencies: - job_info_ext = {} + job_info = copy.deepcopy(self.job_info) - else: - job_info_ext = { - # Asset dependency to wait for at least the scene file to sync. - "AssetDependency0": data["filepath"], - } - - renderer = self._instance.data["renderer"] + if self.asset_dependencies: + # Asset dependency to wait for at least the scene file to sync. + job_info.AssetDependency = self.scene_path # Get layer prefix render_products = self._instance.data["renderProducts"] @@ -569,6 +527,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): # Renderman 22, and so if we are using renderman > 21 we need to set # renderer string on the job to `renderman22`. We will have to change # this when Deadline releases new version handling this. + renderer = self._instance.data["renderer"] if renderer == "renderman": try: from rfm2.config import cfg # noqa @@ -580,29 +539,20 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): renderer = "renderman22" plugin_info = { - "SceneFile": data["filepath"], # Output directory and filename "OutputFilePath": data["dirname"].replace("\\", "/"), "OutputFilePrefix": layer_prefix, - - # Only render layers are considered renderable in this pipeline - "UsingRenderLayers": True, - - # Render only this layer - "RenderLayer": data["renderlayer"], - - # Determine which renderer to use from the file itself - "Renderer": renderer, - - # Resolve relative references - "ProjectPath": data["workspace"], } - payload["JobInfo"].update(job_info_ext) - payload["PluginInfo"].update(plugin_info) - return payload + + return job_info, plugin_info def _get_vray_export_payload(self, data): - payload = copy.deepcopy(self.payload_skeleton) + + job_info = copy.deepcopy(self.job_info) + + job_info.Name = self._job_info_label("Export") + + # Get V-Ray settings info to compute output path vray_settings = cmds.ls(type="VRaySettingsNode") node = vray_settings[0] template = cmds.getAttr("{}.vrscene_filename".format(node)) @@ -610,34 +560,15 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): first_file = self.format_vray_output_filename(scene, template) first_file = "{}/{}".format(data["workspace"], first_file) output = os.path.dirname(first_file) - job_info_ext = { - # Job name, as seen in Monitor - "Name": "Export {} [{}-{}]".format( - data["jobname"], - int(self._instance.data["frameStartHandle"]), - int(self._instance.data["frameEndHandle"])), - "Plugin": self._instance.data.get( - "mayaRenderPlugin", "MayaPype"), - "FramesPerTask": self._instance.data.get("framesPerTask", 1) - } - - plugin_info_ext = { - # Renderer + plugin_info = { "Renderer": "vray", - # Input - "SceneFile": data["filepath"], "SkipExistingFrames": True, - "UsingRenderLayers": True, "UseLegacyRenderLayers": True, - "RenderLayer": data["renderlayer"], - "ProjectPath": data["workspace"], "OutputFilePath": output } - payload["JobInfo"].update(job_info_ext) - payload["PluginInfo"].update(plugin_info_ext) - return payload + return job_info, plugin_info def _get_arnold_export_payload(self, data): @@ -653,76 +584,55 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): script = os.path.normpath(module_path) - payload = copy.deepcopy(self.payload_skeleton) - job_info_ext = { - # Job name, as seen in Monitor - "Name": "Export {} [{}-{}]".format( - data["jobname"], - int(self._instance.data["frameStartHandle"]), - int(self._instance.data["frameEndHandle"])), + job_info = copy.deepcopy(self.job_info) + plugin_info = copy.deepcopy(self.plugin_info) - "Plugin": "Python", - "FramesPerTask": self._instance.data.get("framesPerTask", 1), - "Frames": 1 + job_info.Name = self._job_info_label("Export") + + # Force a single frame Python job + job_info.Plugin = "Python" + job_info.Frames = 1 + + # add required env vars for the export script + envs = { + "AVALON_APP_NAME": os.environ.get("AVALON_APP_NAME"), + "OPENPYPE_ASS_EXPORT_RENDER_LAYER": data["renderlayer"], + "OPENPYPE_ASS_EXPORT_SCENE_FILE": self.scene_path, + "OPENPYPE_ASS_EXPORT_OUTPUT": payload['JobInfo']['OutputFilename0'], # noqa + "OPENPYPE_ASS_EXPORT_START": int(self._instance.data["frameStartHandle"]), # noqa + "OPENPYPE_ASS_EXPORT_END": int(self._instance.data["frameEndHandle"]), # noqa + "OPENPYPE_ASS_EXPORT_STEP": 1 } + for key, value in envs.items(): + job_info.EnvironmentKeyValue = "{key}={value}".format(key=key, + value=value) - plugin_info_ext = { + plugin_info.update({ "Version": "3.6", "ScriptFile": script, "Arguments": "", "SingleFrameOnly": "True", - } - payload["JobInfo"].update(job_info_ext) - payload["PluginInfo"].update(plugin_info_ext) + }) - envs = [ - v - for k, v in payload["JobInfo"].items() - if k.startswith("EnvironmentKeyValue") - ] - - # add app name to environment - envs.append( - "AVALON_APP_NAME={}".format(os.environ.get("AVALON_APP_NAME"))) - envs.append( - "OPENPYPE_ASS_EXPORT_RENDER_LAYER={}".format(data["renderlayer"])) - envs.append( - "OPENPYPE_ASS_EXPORT_SCENE_FILE={}".format(data["filepath"])) - envs.append( - "OPENPYPE_ASS_EXPORT_OUTPUT={}".format( - payload['JobInfo']['OutputFilename0'])) - envs.append( - "OPENPYPE_ASS_EXPORT_START={}".format( - int(self._instance.data["frameStartHandle"]))) - envs.append( - "OPENPYPE_ASS_EXPORT_END={}".format( - int(self._instance.data["frameEndHandle"]))) - envs.append( - "OPENPYPE_ASS_EXPORT_STEP={}".format(1)) - - for i, e in enumerate(envs): - payload["JobInfo"]["EnvironmentKeyValue{}".format(i)] = e - return payload + return job_info, plugin_info def _get_vray_render_payload(self, data): - payload = copy.deepcopy(self.payload_skeleton) + + # Job Info + job_info = copy.deepcopy(self.job_info) + job_info.Name = self._job_info_label("Render") + job_info.Plugin = "Vray" + job_info.OverrideTaskExtraInfoNames = False + + # Plugin Info vray_settings = cmds.ls(type="VRaySettingsNode") node = vray_settings[0] template = cmds.getAttr("{}.vrscene_filename".format(node)) # "vrayscene//_/" - scene, _ = os.path.splitext(data["filename"]) + scene, _ = os.path.splitext(self.scene_path) first_file = self.format_vray_output_filename(scene, template) first_file = "{}/{}".format(data["workspace"], first_file) - job_info_ext = { - "Name": "Render {} [{}-{}]".format( - data["jobname"], - int(self._instance.data["frameStartHandle"]), - int(self._instance.data["frameEndHandle"])), - - "Plugin": "Vray", - "OverrideTaskExtraInfoNames": False, - } plugin_info = { "InputFilename": first_file, @@ -731,35 +641,28 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): "Width": self._instance.data["resolutionWidth"], "Height": self._instance.data["resolutionHeight"], - "OutputFilePath": payload["JobInfo"]["OutputDirectory0"], - "OutputFileName": payload["JobInfo"]["OutputFilename0"] + "OutputFilePath": job_info.OutputDirectory[0], + "OutputFileName": job_info.OutputFilename[0] } - payload["JobInfo"].update(job_info_ext) - payload["PluginInfo"].update(plugin_info) - return payload + return job_info, plugin_info def _get_arnold_render_payload(self, data): - payload = copy.deepcopy(self.payload_skeleton) + + # Job Info + job_info = copy.deepcopy(self.job_info) + job_info.Name = self._job_info_label("Render") + job_info.Plugin = "Arnold" + job_info.OverrideTaskExtraInfoNames = False + + # Plugin Info ass_file, _ = os.path.splitext(data["output_filename_0"]) first_file = ass_file + ".ass" - job_info_ext = { - "Name": "Render {} [{}-{}]".format( - data["jobname"], - int(self._instance.data["frameStartHandle"]), - int(self._instance.data["frameEndHandle"])), - - "Plugin": "Arnold", - "OverrideTaskExtraInfoNames": False, - } - plugin_info = { "ArnoldFile": first_file, } - payload["JobInfo"].update(job_info_ext) - payload["PluginInfo"].update(plugin_info) - return payload + return job_info, plugin_info def format_vray_output_filename(self, filename, template, dir=False): """Format the expected output file of the Export job. @@ -804,7 +707,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): return result - def _patch_workfile(self, file, patches): + def _patch_workfile(self): # type: (str, dict) -> [str, None] """Patch Maya scene. @@ -818,19 +721,25 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): "line": "line to insert" } - Args: - file (str): File to patch. - patches (dict): Dictionary defining patches. - - Returns: - str: Patched file path or None - """ - if not patches or os.path.splitext(file)[1].lower() != ".ma": + project_settings = self._instance.context.data["project_settings"] + patches = ( + project_settings.get( + "deadline", {}).get( + "publish", {}).get( + "MayaSubmitDeadline", {}).get( + "scene_patches", {}) + ) + if not patches: + return + + if not os.path.splitext(self.scene_path)[1].lower() != ".ma": + self.log.debug("Skipping workfile patch since workfile is not " + ".ma file") return compiled_regex = [re.compile(p["regex"]) for p in patches] - with open(file, "r+") as pf: + with open(self.scene_path, "r+") as pf: scene_data = pf.readlines() for ln, line in enumerate(scene_data): for i, r in enumerate(compiled_regex): @@ -839,10 +748,17 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): pf.seek(0) pf.writelines(scene_data) pf.truncate() - self.log.info( - "Applied {} patch to scene.".format( - patches[i]["name"])) - return file + self.log.info("Applied {} patch to scene.".format( + patches[i]["name"] + )) + + def _job_info_label(self, label): + return "{label} {job.Name} [{start}-{end}]".format( + label=label, + job=self.job_info, + start=int(self._instance.data["frameStartHandle"]), + end=int(self._instance.data["frameEndHandle"]), + ) def _format_tiles( From f9bbda244bee373dd3bfb025528923d061808525 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 29 Aug 2022 23:22:19 +0200 Subject: [PATCH 046/122] More explicit PluginInfo name --- .../modules/deadline/plugins/publish/submit_maya_deadline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 8f12a9518f..87ef4e6db9 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -38,7 +38,7 @@ from openpype_modules.deadline.abstract_submit_deadline import DeadlineJobInfo @attr.s -class DeadlinePluginInfo(): +class MayaPluginInfo: SceneFile = attr.ib(default=None) # Input OutputFilePath = attr.ib(default=None) # Output directory and filename OutputFilePrefix = attr.ib(default=None) @@ -208,7 +208,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): instance = self._instance context = instance.context - plugin_info = DeadlinePluginInfo( + plugin_info = MayaPluginInfo( SceneFile=self.scene_path, Version=cmds.about(version=True), RenderLayer=instance.data['setMembers'], From ecf2a89081f19e14c65b0fd7b1992fe80519e983 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 29 Aug 2022 23:39:11 +0200 Subject: [PATCH 047/122] More temp restructuring --- .../plugins/publish/submit_maya_deadline.py | 52 +++++++------------ 1 file changed, 20 insertions(+), 32 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 87ef4e6db9..a77ccd73d4 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -273,9 +273,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): "dirname": dirname, } - # Store output dir for unified publisher (filesequence) - instance.data["outputDir"] = os.path.dirname(output_filename_0) - # Submit preceding export jobs ------------------------------------- export_job = None assert not all(x in instance.data["families"] @@ -326,26 +323,19 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): plugin_info["ImageWidth"] = instance.data.get("resolutionWidth") plugin_info["RegionRendering"] = True - assembly_payload = { - "AuxFiles": [], - "JobInfo": { - "BatchName": payload["JobInfo"]["BatchName"], - "Frames": 1, - "Name": "{} - Tile Assembly Job".format( - payload["JobInfo"]["Name"]), - "OutputDirectory0": - payload["JobInfo"]["OutputDirectory0"].replace( - "\\", "/"), - "Plugin": self.tile_assembler_plugin, - "MachineLimit": 1 - }, - "PluginInfo": { + assembly_job_info = copy.deepcopy(job_info) + assembly_job_info.Plugin = self.tile_assembler_plugin + assembly_job_info.Name = "{job.Name} - Tile Assembly Job".format( + job=job_info) + assembly_job_info.Frames = 1 + assembly_job_info.MachineLimit = 1 + assembly_job_info.Priority = instance.data.get("tile_priority", + self.tile_priority) + + assembly_plugin_info = { "CleanupTiles": 1, "ErrorOnMissing": True - } } - assembly_payload["JobInfo"]["Priority"] = self._instance.data.get( - "tile_priority", self.tile_priority) frame_payloads = [] assembly_payloads = [] @@ -414,6 +404,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): file_index = 1 for file in assembly_files: frame = re.search(R_FRAME_NUMBER, file).group("frame") + new_assembly_payload = copy.deepcopy(assembly_payload) new_assembly_payload["JobInfo"]["Name"] = \ "{} (Frame {})".format( @@ -434,7 +425,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): self.log.info( "Submitting tile job(s) [{}] ...".format(len(frame_payloads))) - url = "{}/api/jobs".format(self.deadline_url) tiles_count = instance.data.get("tilesX") * instance.data.get( "tilesY") # noqa: E501 @@ -444,9 +434,11 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): job_id = response.json()["_id"] hash = response.json()["Props"]["Ex0"] + # Add assembly job dependencies for assembly_job in assembly_payloads: - if assembly_job["JobInfo"]["ExtraInfo0"] == hash: - assembly_job["JobInfo"]["JobDependency0"] = job_id + assembly_job_info = assembly_job["JobInfo"] + if assembly_job_info.ExtraInfo[0] == hash: + assembly_job.JobDependency = job_id for assembly_job in assembly_payloads: file = assembly_job["JobInfo"]["ExtraInfo1"] @@ -461,14 +453,14 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): ) ) + config_file_dir = os.path.dirname(config_file) try: - if not os.path.isdir(os.path.dirname(config_file)): - os.makedirs(os.path.dirname(config_file)) + if not os.path.isdir(config_file_dir): + os.makedirs(config_file_dir) except OSError: # directory is not available - self.log.warning( - "Path is unreachable: `{}`".format( - os.path.dirname(config_file))) + self.log.warning("Path is unreachable: " + "`{}`".format(config_file_dir)) # add config file as job auxFile assembly_job["AuxFiles"] = [config_file] @@ -505,10 +497,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): response.json()["_id"]) job_idx += 1 - instance.data["jobBatchName"] = payload["JobInfo"]["BatchName"] - self.log.info("Setting batch name on instance: {}".format( - instance.data["jobBatchName"])) - def _get_maya_payload(self, data): job_info = copy.deepcopy(self.job_info) From 6abafd0aca1ca06204f5e5bc11907a0a6a855900 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 30 Aug 2022 00:41:30 +0200 Subject: [PATCH 048/122] Refactor tile logic --- .../plugins/publish/submit_maya_deadline.py | 120 ++++++++---------- 1 file changed, 55 insertions(+), 65 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index a77ccd73d4..920adf7e4a 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -314,11 +314,12 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): # if we have sequence of files, we need to create tile job for # every frame - job_info.TileJob = True job_info.TileJobTilesInX = instance.data.get("tilesX") job_info.TileJobTilesInY = instance.data.get("tilesY") + tiles_count = job_info.TileJobTilesInX * job_info.TileJobTilesInY + plugin_info["ImageHeight"] = instance.data.get("resolutionHeight") plugin_info["ImageWidth"] = instance.data.get("resolutionWidth") plugin_info["RegionRendering"] = True @@ -334,7 +335,8 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): assembly_plugin_info = { "CleanupTiles": 1, - "ErrorOnMissing": True + "ErrorOnMissing": True, + "Renderer": self._instance.data["renderer"] } frame_payloads = [] @@ -367,81 +369,69 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): file_index = 1 for file in files: frame = re.search(R_FRAME_NUMBER, file).group("frame") - new_payload = copy.deepcopy(payload) - new_payload["JobInfo"]["Name"] = \ - "{} (Frame {} - {} tiles)".format( + + new_job_info = copy.deepcopy(job_info) + new_job_info.Name = "{} (Frame {} - {} tiles)".format( payload["JobInfo"]["Name"], frame, instance.data.get("tilesX") * instance.data.get("tilesY") - # noqa: E501 - ) - self.log.info( - "... preparing job {}".format( - new_payload["JobInfo"]["Name"])) - new_payload["JobInfo"]["TileJobFrame"] = frame + ) + new_job_info.TileJobFrame = frame - tiles_data = _format_tiles( + new_plugin_info = copy.deepcopy(plugin_info) + + # Add tile data into job info and plugin info + tiles_out, _ = _format_tiles( file, 0, instance.data.get("tilesX"), instance.data.get("tilesY"), instance.data.get("resolutionWidth"), instance.data.get("resolutionHeight"), payload["PluginInfo"]["OutputFilePrefix"] - )[0] - new_payload["JobInfo"].update(tiles_data["JobInfo"]) - new_payload["PluginInfo"].update(tiles_data["PluginInfo"]) + ) + new_job_info.update(tiles_out["JobInfo"]) + new_plugin_info.update(tiles_out["PluginInfo"]) self.log.info("hashing {} - {}".format(file_index, file)) job_hash = hashlib.sha256( ("{}_{}".format(file_index, file)).encode("utf-8")) frame_jobs[frame] = job_hash.hexdigest() - new_payload["JobInfo"]["ExtraInfo0"] = job_hash.hexdigest() - new_payload["JobInfo"]["ExtraInfo1"] = file - frame_payloads.append(new_payload) - file_index += 1 + new_job_info.ExtraInfo[0] = job_hash.hexdigest() + new_job_info.ExtraInfo[1] = file - file_index = 1 - for file in assembly_files: - frame = re.search(R_FRAME_NUMBER, file).group("frame") - - new_assembly_payload = copy.deepcopy(assembly_payload) - new_assembly_payload["JobInfo"]["Name"] = \ - "{} (Frame {})".format( - assembly_payload["JobInfo"]["Name"], - frame) - new_assembly_payload["JobInfo"]["OutputFilename0"] = re.sub( - REPL_FRAME_NUMBER, - "\\1{}\\3".format("#" * len(frame)), file) - - new_assembly_payload["PluginInfo"]["Renderer"] = \ - self._instance.data["renderer"] # noqa: E501 - new_assembly_payload["JobInfo"]["ExtraInfo0"] = frame_jobs[ - frame] # noqa: E501 - new_assembly_payload["JobInfo"]["ExtraInfo1"] = file - assembly_payloads.append(new_assembly_payload) + frame_payloads.append(self.assemble_payload( + job_info=new_job_info, + plugin_info=new_plugin_info + )) file_index += 1 self.log.info( "Submitting tile job(s) [{}] ...".format(len(frame_payloads))) - tiles_count = instance.data.get("tilesX") * instance.data.get( - "tilesY") # noqa: E501 - - for tile_job in frame_payloads: - response = self.submit(tile_job) - + frame_tile_job_id = {} + for tile_job_payload in frame_payloads: + response = self.submit(tile_job_payload) job_id = response.json()["_id"] - hash = response.json()["Props"]["Ex0"] + frame_tile_job_id[frame] = job_id - # Add assembly job dependencies - for assembly_job in assembly_payloads: - assembly_job_info = assembly_job["JobInfo"] - if assembly_job_info.ExtraInfo[0] == hash: - assembly_job.JobDependency = job_id + assembly_jobs = [] + for i, file in enumerate(assembly_files): + frame = re.search(R_FRAME_NUMBER, file).group("frame") + + frame_assembly_job_info = copy.deepcopy(assembly_job_info) + frame_assembly_job_info.Name += " (Frame {})".format(frame) + frame_assembly_job_info.OutputFilename[0] = re.sub( + REPL_FRAME_NUMBER, + "\\1{}\\3".format("#" * len(frame)), file) + + hash = frame_jobs[frame] + tile_job_id = frame_tile_job_id[frame] + + frame_assembly_job_info.ExtraInfo[0] = hash + frame_assembly_job_info.ExtraInfo[1] = file + frame_assembly_job_info.JobDependency = tile_job_id - for assembly_job in assembly_payloads: - file = assembly_job["JobInfo"]["ExtraInfo1"] # write assembly job config files now = datetime.now() @@ -462,9 +452,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): self.log.warning("Path is unreachable: " "`{}`".format(config_file_dir)) - # add config file as job auxFile - assembly_job["AuxFiles"] = [config_file] - with open(config_file, "w") as cf: print("TileCount={}".format(tiles_count), file=cf) print("ImageFileName={}".format(file), file=cf) @@ -485,17 +472,20 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): for k, v in tiles.items(): print("{}={}".format(k, v), file=cf) - job_idx = 1 - instance.data["assemblySubmissionJobs"] = [] - for ass_job in assembly_payloads: - self.log.info("submitting assembly job {} of {}".format( - job_idx, len(assembly_payloads) - )) - response = self.submit(ass_job) + payload = self.assemble_payload( + job_info=frame_assembly_job_info, + plugin_info=assembly_plugin_info.copy(), + # add config file as job auxFile + aux_files=[config_file] + ) - instance.data["assemblySubmissionJobs"].append( - response.json()["_id"]) - job_idx += 1 + self.log.info("submitting assembly job {} of {}".format( + i+1, len(assembly_payloads) + )) + response = self.submit(payload) + assembly_jobs.append(response.json()["_id"]) + + instance.data["assemblySubmissionJobs"] = assembly_jobs def _get_maya_payload(self, data): From a6002de641e1ad500192be433f620f17680ea056 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 30 Aug 2022 00:58:23 +0200 Subject: [PATCH 049/122] Refactor _format_tiles for readability --- .../plugins/publish/submit_maya_deadline.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 920adf7e4a..00d8eb7859 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -800,23 +800,23 @@ def _format_tiles( tiles_x, tiles_y ) - out_tile_index = "OutputFilename{}Tile{}".format( - str(index), tile - ) + + # Job Info new_filename = "{}/{}{}".format( os.path.dirname(filename), tile_prefix, os.path.basename(filename) ) - out["JobInfo"][out_tile_index] = new_filename - out["PluginInfo"]["RegionPrefix{}".format(tile)] = \ - "/{}".format(tile_prefix).join(prefix.rsplit("/", 1)) + out["JobInfo"]["OutputFilename{}Tile{}".format(index, tile)] = new_filename # noqa + # Plugin Info + out["PluginInfo"]["RegionPrefix{}".format(tile)] = "/{}".format(tile_prefix).join(prefix.rsplit("/", 1)) # noqa: E501 out["PluginInfo"]["RegionTop{}".format(tile)] = int(height) - (tile_y * h_space) # noqa: E501 out["PluginInfo"]["RegionBottom{}".format(tile)] = int(height) - ((tile_y - 1) * h_space) - 1 # noqa: E501 out["PluginInfo"]["RegionLeft{}".format(tile)] = (tile_x - 1) * w_space # noqa: E501 out["PluginInfo"]["RegionRight{}".format(tile)] = (tile_x * w_space) - 1 # noqa: E501 + # Tile config cfg["Tile{}".format(tile)] = new_filename cfg["Tile{}Tile".format(tile)] = new_filename cfg["Tile{}FileName".format(tile)] = new_filename @@ -828,4 +828,5 @@ def _format_tiles( cfg["Tile{}Height".format(tile)] = h_space tile += 1 + return out, cfg From a9fe806fec1a5e5ecf98327cfa4845b8b6d3edc0 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 30 Aug 2022 01:02:24 +0200 Subject: [PATCH 050/122] Calculate once --- .../plugins/publish/submit_maya_deadline.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 00d8eb7859..d0348119dc 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -800,6 +800,10 @@ def _format_tiles( tiles_x, tiles_y ) + top = int(height) - (tile_y * h_space) + bottom = int(height) - ((tile_y - 1) * h_space) - 1 + left = (tile_x - 1) * w_space + right = (tile_x * w_space) - 1 # Job Info new_filename = "{}/{}{}".format( @@ -811,19 +815,17 @@ def _format_tiles( # Plugin Info out["PluginInfo"]["RegionPrefix{}".format(tile)] = "/{}".format(tile_prefix).join(prefix.rsplit("/", 1)) # noqa: E501 - out["PluginInfo"]["RegionTop{}".format(tile)] = int(height) - (tile_y * h_space) # noqa: E501 - out["PluginInfo"]["RegionBottom{}".format(tile)] = int(height) - ((tile_y - 1) * h_space) - 1 # noqa: E501 - out["PluginInfo"]["RegionLeft{}".format(tile)] = (tile_x - 1) * w_space # noqa: E501 - out["PluginInfo"]["RegionRight{}".format(tile)] = (tile_x * w_space) - 1 # noqa: E501 + out["PluginInfo"]["RegionTop{}".format(tile)] = top + out["PluginInfo"]["RegionBottom{}".format(tile)] = bottom + out["PluginInfo"]["RegionLeft{}".format(tile)] = left + out["PluginInfo"]["RegionRight{}".format(tile)] = right # Tile config cfg["Tile{}".format(tile)] = new_filename cfg["Tile{}Tile".format(tile)] = new_filename cfg["Tile{}FileName".format(tile)] = new_filename - cfg["Tile{}X".format(tile)] = (tile_x - 1) * w_space - - cfg["Tile{}Y".format(tile)] = int(height) - (tile_y * h_space) - + cfg["Tile{}X".format(tile)] = left + cfg["Tile{}Y".format(tile)] = top cfg["Tile{}Width".format(tile)] = w_space cfg["Tile{}Height".format(tile)] = h_space From d7c72f97b30f85aca15b4a8148c140595f0b2a3a Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 30 Aug 2022 01:10:19 +0200 Subject: [PATCH 051/122] Batch submit assembly jobs --- .../plugins/publish/submit_maya_deadline.py | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index d0348119dc..265c0f79ec 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -339,7 +339,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): "Renderer": self._instance.data["renderer"] } - frame_payloads = [] assembly_payloads = [] R_FRAME_NUMBER = re.compile( @@ -358,14 +357,15 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): itertools.chain.from_iterable( [f for _, f in exp[0].items()])) if not files: - # if beauty doesn't exists, use first aov we found + # if beauty doesn't exist, use first aov we found files = exp[0].get(list(exp[0].keys())[0]) else: files = exp assembly_files = files + # Define frame tile jobs frame_jobs = {} - + frame_payloads = {} file_index = 1 for file in files: frame = re.search(R_FRAME_NUMBER, file).group("frame") @@ -400,22 +400,24 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): new_job_info.ExtraInfo[0] = job_hash.hexdigest() new_job_info.ExtraInfo[1] = file - frame_payloads.append(self.assemble_payload( + frame_payloads[frame] = self.assemble_payload( job_info=new_job_info, plugin_info=new_plugin_info - )) + ) file_index += 1 self.log.info( "Submitting tile job(s) [{}] ...".format(len(frame_payloads))) + # Submit frame tile jobs frame_tile_job_id = {} - for tile_job_payload in frame_payloads: + for frame, tile_job_payload in frame_payloads.items(): response = self.submit(tile_job_payload) job_id = response.json()["_id"] frame_tile_job_id[frame] = job_id - assembly_jobs = [] + # Define assembly payloads + assembly_payloads = [] for i, file in enumerate(assembly_files): frame = re.search(R_FRAME_NUMBER, file).group("frame") @@ -478,14 +480,18 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): # add config file as job auxFile aux_files=[config_file] ) + assembly_payloads.append(payload) + # Submit assembly jobs + assembly_job_ids = [] + for i, payload in enumerate(assembly_payloads): self.log.info("submitting assembly job {} of {}".format( i+1, len(assembly_payloads) )) response = self.submit(payload) - assembly_jobs.append(response.json()["_id"]) + assembly_job_ids.append(response.json()["_id"]) - instance.data["assemblySubmissionJobs"] = assembly_jobs + instance.data["assemblySubmissionJobs"] = assembly_job_ids def _get_maya_payload(self, data): From 965522585b98e441907480caee57af5dad92c2d2 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 30 Aug 2022 01:11:12 +0200 Subject: [PATCH 052/122] Remove redundant docstring --- .../deadline/plugins/publish/submit_maya_deadline.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 265c0f79ec..cd9f426977 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -51,16 +51,6 @@ class MayaPluginInfo: class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): - """Submit available render layers to Deadline. - - Renders are submitted to a Deadline Web Service as - supplied via settings key "DEADLINE_REST_URL". - - Attributes: - use_published (bool): Use published scene to render instead of the - one in work area. - - """ label = "Submit Render to Deadline" hosts = ["maya"] From 8af88e115723e6abc73fff279773043e4a520326 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 30 Aug 2022 09:08:44 +0200 Subject: [PATCH 053/122] More cleanup --- .../plugins/publish/submit_maya_deadline.py | 98 +++++++------------ 1 file changed, 38 insertions(+), 60 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index cd9f426977..95140a082f 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -86,8 +86,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): job_info.Name = "%s - %s" % (src_filename, instance.name) job_info.BatchName = src_filename job_info.Plugin = instance.data.get("mayaRenderPlugin", "MayaBatch") - job_info.UserName = context.data.get( - "deadlineUser", getpass.getuser()) + job_info.UserName = context.data.get("deadlineUser", getpass.getuser()) # Deadline requires integers in frame range frames = "{start}-{end}x{step}".format( @@ -134,25 +133,18 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): environment = dict({key: os.environ[key] for key in keys if key in os.environ}, **legacy_io.Session) - # TODO: Taken from old publish class - test whether still needed - environment["OPENPYPE_LOG_NO_COLORS"] = "1" # to recognize job from PYPE for turning Event On/Off environment["OPENPYPE_RENDER_JOB"] = "1" + environment["OPENPYPE_LOG_NO_COLORS"] = "1" - for key in keys: - val = environment.get(key) - if val: - job_info.EnvironmentKeyValue = "{key}={value}".format( - key=key, - value=val - ) - # to recognize job from PYPE for turning Event On/Off - job_info.EnvironmentKeyValue = "OPENPYPE_RENDER_JOB=1" - job_info.EnvironmentKeyValue = "OPENPYPE_LOG_NO_COLORS=1" + for key, value in environment.items(): + if not value: + continue + job_info.EnvironmentKeyValue = "{key}={value}".format(key=key, + value=value) - # Optional, enable double-click to preview rendered - # frames from Deadline Monitor - for i, filepath in enumerate(instance.data["files"]): + # Enable double-click to preview rendered frames from Deadline Monitor + for filepath in instance.data["files"]: dirname = os.path.dirname(filepath) fname = os.path.basename(filepath) job_info.OutputDirectory = dirname.replace("\\", "/") @@ -241,25 +233,10 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): filename = os.path.basename(filepath) dirname = os.path.join(workspace, default_render_file) - # this is needed because renderman handles directory and file - # prefixes separately - if self._instance.data["renderer"] == "renderman": - dirname = os.path.dirname(output_filename_0) - - # Create render folder ---------------------------------------------- - try: - # Ensure render folder exists - os.makedirs(dirname) - except OSError: - pass - # Fill in common data to payload ------------------------------------ # TODO: Replace these with collected data from CollectRender payload_data = { "filename": filename, - "filepath": filepath, - "output_filename_0": output_filename_0, - "renderlayer": renderlayer, "dirname": dirname, } @@ -299,8 +276,8 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): def _tile_render(self, instance, payload): # As collected by super process() - job_info = self.job_info - plugin_info = self.pluginInfo + job_info = copy.deepcopy(self.job_info) + plugin_info = copy.deepcopy(self.plugin_info) # if we have sequence of files, we need to create tile job for # every frame @@ -314,23 +291,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): plugin_info["ImageWidth"] = instance.data.get("resolutionWidth") plugin_info["RegionRendering"] = True - assembly_job_info = copy.deepcopy(job_info) - assembly_job_info.Plugin = self.tile_assembler_plugin - assembly_job_info.Name = "{job.Name} - Tile Assembly Job".format( - job=job_info) - assembly_job_info.Frames = 1 - assembly_job_info.MachineLimit = 1 - assembly_job_info.Priority = instance.data.get("tile_priority", - self.tile_priority) - - assembly_plugin_info = { - "CleanupTiles": 1, - "ErrorOnMissing": True, - "Renderer": self._instance.data["renderer"] - } - - assembly_payloads = [] - R_FRAME_NUMBER = re.compile( r".+\.(?P[0-9]+)\..+") # noqa: N806, E501 REPL_FRAME_NUMBER = re.compile( @@ -407,7 +367,23 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): frame_tile_job_id[frame] = job_id # Define assembly payloads + assembly_job_info = copy.deepcopy(job_info) + assembly_job_info.Plugin = self.tile_assembler_plugin + assembly_job_info.Name = "{job.Name} - Tile Assembly Job".format( + job=job_info) + assembly_job_info.Frames = 1 + assembly_job_info.MachineLimit = 1 + assembly_job_info.Priority = instance.data.get("tile_priority", + self.tile_priority) + + assembly_plugin_info = { + "CleanupTiles": 1, + "ErrorOnMissing": True, + "Renderer": self._instance.data["renderer"] + } + assembly_payloads = [] + output_dir = self.job_info.OutputDirectory[0] for i, file in enumerate(assembly_files): frame = re.search(R_FRAME_NUMBER, file).group("frame") @@ -427,22 +403,19 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): # write assembly job config files now = datetime.now() - config_file = os.path.join( - os.path.dirname(output_filename_0), + config_file = os.path.join(output_dir, "{}_config_{}.txt".format( os.path.splitext(file)[0], now.strftime("%Y_%m_%d_%H_%M_%S") ) ) - - config_file_dir = os.path.dirname(config_file) try: - if not os.path.isdir(config_file_dir): - os.makedirs(config_file_dir) + if not os.path.isdir(output_dir): + os.makedirs(output_dir) except OSError: # directory is not available self.log.warning("Path is unreachable: " - "`{}`".format(config_file_dir)) + "`{}`".format(output_dir)) with open(config_file, "w") as cf: print("TileCount={}".format(tiles_count), file=cf) @@ -567,17 +540,22 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): job_info.Plugin = "Python" job_info.Frames = 1 + renderlayer = self._instance.data["setMembers"] + # add required env vars for the export script envs = { "AVALON_APP_NAME": os.environ.get("AVALON_APP_NAME"), - "OPENPYPE_ASS_EXPORT_RENDER_LAYER": data["renderlayer"], + "OPENPYPE_ASS_EXPORT_RENDER_LAYER": renderlayer, "OPENPYPE_ASS_EXPORT_SCENE_FILE": self.scene_path, - "OPENPYPE_ASS_EXPORT_OUTPUT": payload['JobInfo']['OutputFilename0'], # noqa + "OPENPYPE_ASS_EXPORT_OUTPUT": job_info.OutputFilename[0], "OPENPYPE_ASS_EXPORT_START": int(self._instance.data["frameStartHandle"]), # noqa "OPENPYPE_ASS_EXPORT_END": int(self._instance.data["frameEndHandle"]), # noqa "OPENPYPE_ASS_EXPORT_STEP": 1 } for key, value in envs.items(): + if not value: + continue + job_info.EnvironmentKeyValue = "{key}={value}".format(key=key, value=value) From e8aa926cb7d338427ce7ba558f8ffa1609fde8ef Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 30 Aug 2022 09:28:42 +0200 Subject: [PATCH 054/122] Move single use of in-line function to the class for readability --- .../deadline/abstract_submit_deadline.py | 39 ++++++++++--------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py index f56cf49f6d..a3db3feac9 100644 --- a/openpype/modules/deadline/abstract_submit_deadline.py +++ b/openpype/modules/deadline/abstract_submit_deadline.py @@ -544,26 +544,8 @@ class AbstractSubmitDeadline(pyblish.api.InstancePlugin): """ - def _get_workfile_instance(context): - """Find workfile instance in context""" - for i in context: - - is_workfile = ( - "workfile" in i.data.get("families", []) or - i.data["family"] == "workfile" - ) - if not is_workfile: - continue - - # test if there is instance of workfile waiting - # to be published. - assert i.data["publish"] is True, ( - "Workfile (scene) must be published along") - - return i - instance = self._instance - workfile_instance = _get_workfile_instance(instance.context) + workfile_instance = self._get_workfile_instance(instance.context) if not workfile_instance: return @@ -689,3 +671,22 @@ class AbstractSubmitDeadline(pyblish.api.InstancePlugin): self._instance.data["deadlineSubmissionJob"] = result return result["_id"] + + @staticmethod + def _get_workfile_instance(context): + """Find workfile instance in context""" + for i in context: + + is_workfile = ( + "workfile" in i.data.get("families", []) or + i.data["family"] == "workfile" + ) + if not is_workfile: + continue + + # test if there is instance of workfile waiting + # to be published. + assert i.data["publish"] is True, ( + "Workfile (scene) must be published along") + + return i From c6a0a199e1810e5c63c484d5871d34d14594e5be Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 30 Aug 2022 09:40:08 +0200 Subject: [PATCH 055/122] Cosmetics --- .../deadline/plugins/publish/submit_maya_deadline.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 95140a082f..873005e051 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -267,15 +267,16 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): if instance.data.get("tileRendering"): # Prepare tiles data - self._tile_render(instance, payload) + self._tile_render(payload) else: # Submit main render job job_info, plugin_info = payload self.submit(self.assemble_payload(job_info, plugin_info)) - def _tile_render(self, instance, payload): + def _tile_render(self, payload): # As collected by super process() + instance = self._instance job_info = copy.deepcopy(self.job_info) plugin_info = copy.deepcopy(self.plugin_info) @@ -321,11 +322,8 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): frame = re.search(R_FRAME_NUMBER, file).group("frame") new_job_info = copy.deepcopy(job_info) - new_job_info.Name = "{} (Frame {} - {} tiles)".format( - payload["JobInfo"]["Name"], - frame, - instance.data.get("tilesX") * instance.data.get("tilesY") - ) + new_job_info.Name += " (Frame {} - {} tiles)".format(frame, + tiles_count) new_job_info.TileJobFrame = frame new_plugin_info = copy.deepcopy(plugin_info) From e429e2ec41d23c097b301c31485a38aea634d6a3 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 30 Aug 2022 10:19:38 +0200 Subject: [PATCH 056/122] Remove json dump since `renderProducts` are not serializable --- openpype/hosts/maya/plugins/publish/collect_render.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openpype/hosts/maya/plugins/publish/collect_render.py b/openpype/hosts/maya/plugins/publish/collect_render.py index 768a53329f..14aac2f206 100644 --- a/openpype/hosts/maya/plugins/publish/collect_render.py +++ b/openpype/hosts/maya/plugins/publish/collect_render.py @@ -360,7 +360,6 @@ class CollectMayaRender(pyblish.api.ContextPlugin): instance.data["label"] = label instance.data["farm"] = True instance.data.update(data) - self.log.debug("data: {}".format(json.dumps(data, indent=4))) def parse_options(self, render_globals): """Get all overrides with a value, skip those without. From 9472cbe271af4be7c0328bf05200f44c521c58ae Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 30 Aug 2022 11:23:14 +0200 Subject: [PATCH 057/122] Fix submission --- .../collect_deadline_server_from_instance.py | 2 +- .../plugins/publish/submit_maya_deadline.py | 71 +++++++++---------- 2 files changed, 33 insertions(+), 40 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/collect_deadline_server_from_instance.py b/openpype/modules/deadline/plugins/publish/collect_deadline_server_from_instance.py index a7035cd99f..9981bead3e 100644 --- a/openpype/modules/deadline/plugins/publish/collect_deadline_server_from_instance.py +++ b/openpype/modules/deadline/plugins/publish/collect_deadline_server_from_instance.py @@ -13,7 +13,7 @@ class CollectDeadlineServerFromInstance(pyblish.api.InstancePlugin): order = pyblish.api.CollectorOrder + 0.415 label = "Deadline Webservice from the Instance" - families = ["rendering"] + families = ["rendering", "renderlayer"] def process(self, instance): instance.data["deadlineUrl"] = self._collect_deadline_url(instance) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 873005e051..2afa1883c4 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -27,7 +27,6 @@ import itertools from collections import OrderedDict import attr -import clique from maya import cmds @@ -111,7 +110,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): # Add options from RenderGlobals render_globals = instance.data.get("renderGlobals", {}) - for key, value in render_globals: + for key, value in render_globals.items(): setattr(job_info, key, value) keys = [ @@ -143,13 +142,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): job_info.EnvironmentKeyValue = "{key}={value}".format(key=key, value=value) - # Enable double-click to preview rendered frames from Deadline Monitor - for filepath in instance.data["files"]: - dirname = os.path.dirname(filepath) - fname = os.path.basename(filepath) - job_info.OutputDirectory = dirname.replace("\\", "/") - job_info.OutputFilename = fname - # Adding file dependencies. if self.asset_dependencies: dependencies = instance.context.data["fileDependencies"] @@ -160,28 +152,9 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): # Add list of expected files to job # --------------------------------- exp = instance.data.get("expectedFiles") - - def _get_output_filename(files): - col, rem = clique.assemble(files) - if not col and rem: - # we couldn't find any collections but have - # individual files. - assert len(rem) == 1, ( - "Found multiple non related files " - "to render, don't know what to do " - "with them.") - return rem[0] - else: - return col[0].format('{head}{padding}{tail}') - - if isinstance(exp[0], dict): - # we have aovs and we need to iterate over them - for _aov, files in exp[0].items(): - output_file = _get_output_filename(files) - job_info.OutputFilename = output_file - else: - output_file = _get_output_filename(exp) - job_info.OutputFilename = output_file + for filepath in self._iter_expected_files(exp): + job_info.OutputDirectory = os.path.dirname(filepath) + job_info.OutputFilename = os.path.basename(filepath) return job_info @@ -194,6 +167,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): SceneFile=self.scene_path, Version=cmds.about(version=True), RenderLayer=instance.data['setMembers'], + Renderer=instance.data["renderer"], RenderSetupIncludeLights=instance.data.get("renderSetupIncludeLights"), # noqa ProjectPath=context.data["workspaceDir"], UsingRenderLayers=True, @@ -216,7 +190,9 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): # TODO: Avoid the need for this logic here, needed for submit publish # Store output dir for unified publisher (filesequence) - output_dir = os.path.dirname(instance.data["files"][0]) + expected_files = instance.data["expectedFiles"] + first_file = next(self._iter_expected_files(expected_files)) + output_dir = os.path.dirname(first_file) instance.data["outputDir"] = output_dir instance.data["toBeRenderedOn"] = "deadline" @@ -247,17 +223,20 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): "Vray Scene and Ass Scene options are mutually exclusive") if "vrayscene" in instance.data["families"]: + self.log.debug("Submitting V-Ray scene render..") vray_export_payload = self._get_vray_export_payload(payload_data) export_job = self.submit(vray_export_payload) payload = self._get_vray_render_payload(payload_data) elif "assscene" in instance.data["families"]: + self.log.debug("Submitting Arnold .ass standalone render..") ass_export_payload = self._get_arnold_export_payload(payload_data) export_job = self.submit(ass_export_payload) payload = self._get_arnold_render_payload(payload_data) else: + self.log.debug("Submitting MayaBatch render..") payload = self._get_maya_payload(payload_data) # Add export job as dependency -------------------------------------- @@ -274,6 +253,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): self.submit(self.assemble_payload(job_info, plugin_info)) def _tile_render(self, payload): + """Submit as tile render per frame with dependent assembly jobs.""" # As collected by super process() instance = self._instance @@ -315,7 +295,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): assembly_files = files # Define frame tile jobs - frame_jobs = {} + frame_file_hash = {} frame_payloads = {} file_index = 1 for file in files: @@ -343,9 +323,11 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): self.log.info("hashing {} - {}".format(file_index, file)) job_hash = hashlib.sha256( ("{}_{}".format(file_index, file)).encode("utf-8")) - frame_jobs[frame] = job_hash.hexdigest() - new_job_info.ExtraInfo[0] = job_hash.hexdigest() + file_hash = job_hash.hexdigest() + frame_file_hash[frame] = file_hash + + new_job_info.ExtraInfo[0] = file_hash new_job_info.ExtraInfo[1] = file frame_payloads[frame] = self.assemble_payload( @@ -391,10 +373,10 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): REPL_FRAME_NUMBER, "\\1{}\\3".format("#" * len(frame)), file) - hash = frame_jobs[frame] + file_hash = frame_file_hash[frame] tile_job_id = frame_tile_job_id[frame] - frame_assembly_job_info.ExtraInfo[0] = hash + frame_assembly_job_info.ExtraInfo[0] = file_hash frame_assembly_job_info.ExtraInfo[1] = file frame_assembly_job_info.JobDependency = tile_job_id @@ -483,11 +465,12 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): if int(rman_version.split(".")[0]) > 22: renderer = "renderman22" - plugin_info = { + plugin_info = copy.deepcopy(self.plugin_info) + plugin_info.update({ # Output directory and filename "OutputFilePath": data["dirname"].replace("\\", "/"), "OutputFilePrefix": layer_prefix, - } + }) return job_info, plugin_info @@ -710,6 +693,16 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): end=int(self._instance.data["frameEndHandle"]), ) + @staticmethod + def _iter_expected_files(exp): + if isinstance(exp[0], dict): + for _aov, files in exp[0].items(): + for file in files: + yield file + else: + for file in exp: + yield file + def _format_tiles( filename, index, tiles_x, tiles_y, From 227b8405479e3a87b507e63a9b59fe473d7c9276 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 30 Aug 2022 14:12:54 +0200 Subject: [PATCH 058/122] Refactor AbstractSubmitDeadline vars to allow easier access to indices --- .../deadline/abstract_submit_deadline.py | 251 +++++++----------- .../plugins/publish/submit_maya_deadline.py | 35 ++- 2 files changed, 115 insertions(+), 171 deletions(-) diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py index a3db3feac9..427faec115 100644 --- a/openpype/modules/deadline/abstract_submit_deadline.py +++ b/openpype/modules/deadline/abstract_submit_deadline.py @@ -9,6 +9,7 @@ import os from abc import abstractmethod import platform import getpass +from functools import partial from collections import OrderedDict import six @@ -66,6 +67,58 @@ def requests_get(*args, **kwargs): return requests.get(*args, **kwargs) +class DeadlineIndexedVar(dict): + """ + + Allows to set and query values by integer indices: + Query: var[1] or var.get(1) + Set: var[1] = "my_value" + Append: var += "value" + + Note: Iterating the instance is not guarantueed to be the order of the + indices. To do so iterate with `sorted()` + + """ + def __init__(self, key): + self.__key = key + + def next_available_index(self): + # Add as first unused entry + i = 0 + while i in self.keys(): + i += 1 + return i + + def serialize(self): + key = self.__key + + # Allow custom location for index in serialized string + if "{}" not in key: + key = key + "{}" + + return { + key.format(index): value for index, value in sorted(self.items()) + } + + def update(self, data): + # Force the integer key check + for key, value in data.items(): + self.__setitem__(key, value) + + def __iadd__(self, other): + index = self.next_available_index() + self[index] = other + return self + + def __setitem__(self, key, value): + if not isinstance(key, int): + raise TypeError("Key must be an integer: {}".format(key)) + + if key < 0: + raise ValueError("Negative index can't be set: {}".format(key)) + dict.__setitem__(self, key, value) + + @attr.s class DeadlineJobInfo(object): """Mapping of all Deadline *JobInfo* attributes. @@ -218,24 +271,8 @@ class DeadlineJobInfo(object): # Environment # ---------------------------------------------- - _environmentKeyValue = attr.ib(factory=list) - - @property - def EnvironmentKeyValue(self): # noqa: N802 - """Return all environment key values formatted for Deadline. - - Returns: - dict: as `{'EnvironmentKeyValue0', 'key=value'}` - - """ - out = {} - for index, v in enumerate(self._environmentKeyValue): - out["EnvironmentKeyValue{}".format(index)] = v - return out - - @EnvironmentKeyValue.setter - def EnvironmentKeyValue(self, val): # noqa: N802 - self._environmentKeyValue.append(val) + EnvironmentKeyValue = attr.ib(factory=partial(DeadlineIndexedVar, + "EnvironmentKeyValue")) IncludeEnvironment = attr.ib(default=None) # Default: false UseJobEnvironmentOnly = attr.ib(default=None) # Default: false @@ -243,142 +280,29 @@ class DeadlineJobInfo(object): # Job Extra Info # ---------------------------------------------- - _extraInfos = attr.ib(factory=list) - _extraInfoKeyValues = attr.ib(factory=list) - - @property - def ExtraInfo(self): # noqa: N802 - """Return all ExtraInfo values formatted for Deadline. - - Returns: - dict: as `{'ExtraInfo0': 'value'}` - - """ - out = {} - for index, v in enumerate(self._extraInfos): - out["ExtraInfo{}".format(index)] = v - return out - - @ExtraInfo.setter - def ExtraInfo(self, val): # noqa: N802 - self._extraInfos.append(val) - - @property - def ExtraInfoKeyValue(self): # noqa: N802 - """Return all ExtraInfoKeyValue values formatted for Deadline. - - Returns: - dict: as {'ExtraInfoKeyValue0': 'key=value'}` - - """ - out = {} - for index, v in enumerate(self._extraInfoKeyValues): - out["ExtraInfoKeyValue{}".format(index)] = v - return out - - @ExtraInfoKeyValue.setter - def ExtraInfoKeyValue(self, val): # noqa: N802 - self._extraInfoKeyValues.append(val) + ExtraInfo = attr.ib(factory=partial(DeadlineIndexedVar, "ExtraInfo")) + ExtraInfoKeyValue = attr.ib(factory=partial(DeadlineIndexedVar, + "ExtraInfoKeyValue")) # Task Extra Info Names # ---------------------------------------------- OverrideTaskExtraInfoNames = attr.ib(default=None) # Default: false - _taskExtraInfos = attr.ib(factory=list) - - @property - def TaskExtraInfoName(self): # noqa: N802 - """Return all TaskExtraInfoName values formatted for Deadline. - - Returns: - dict: as `{'TaskExtraInfoName0': 'value'}` - - """ - out = {} - for index, v in enumerate(self._taskExtraInfos): - out["TaskExtraInfoName{}".format(index)] = v - return out - - @TaskExtraInfoName.setter - def TaskExtraInfoName(self, val): # noqa: N802 - self._taskExtraInfos.append(val) + TaskExtraInfoName = attr.ib(factory=partial(DeadlineIndexedVar, + "TaskExtraInfoName")) # Output # ---------------------------------------------- - _outputFilename = attr.ib(factory=list) - _outputFilenameTile = attr.ib(factory=list) - _outputDirectory = attr.ib(factory=list) - - @property - def OutputFilename(self): # noqa: N802 - """Return all OutputFilename values formatted for Deadline. - - Returns: - dict: as `{'OutputFilename0': 'filename'}` - - """ - out = {} - for index, v in enumerate(self._outputFilename): - out["OutputFilename{}".format(index)] = v - return out - - @OutputFilename.setter - def OutputFilename(self, val): # noqa: N802 - self._outputFilename.append(val) - - @property - def OutputFilenameTile(self): # noqa: N802 - """Return all OutputFilename#Tile values formatted for Deadline. - - Returns: - dict: as `{'OutputFilenme#Tile': 'tile'}` - - """ - out = {} - for index, v in enumerate(self._outputFilenameTile): - out["OutputFilename{}Tile".format(index)] = v - return out - - @OutputFilenameTile.setter - def OutputFilenameTile(self, val): # noqa: N802 - self._outputFilenameTile.append(val) - - @property - def OutputDirectory(self): # noqa: N802 - """Return all OutputDirectory values formatted for Deadline. - - Returns: - dict: as `{'OutputDirectory0': 'dir'}` - - """ - out = {} - for index, v in enumerate(self._outputDirectory): - out["OutputDirectory{}".format(index)] = v - return out - - @OutputDirectory.setter - def OutputDirectory(self, val): # noqa: N802 - self._outputDirectory.append(val) + OutputFilename = attr.ib(factory=partial(DeadlineIndexedVar, + "OutputFilename")) + OutputFilenameTile = attr.ib(factory=partial(DeadlineIndexedVar, + "OutputFilename{}Tile")) + OutputDirectory = attr.ib(factory=partial(DeadlineIndexedVar, + "OutputDirectory")) # Asset Dependency # ---------------------------------------------- - _assetDependency = attr.ib(factory=list) - - @property - def AssetDependency(self): # noqa: N802 - """Return all OutputDirectory values formatted for Deadline. - - Returns: - dict: as `{'OutputDirectory0': 'dir'}` - - """ - out = {} - for index, v in enumerate(self._assetDependency): - out["AssetDependency{}".format(index)] = v - return out - - @OutputDirectory.setter - def AssetDependency(self, val): # noqa: N802 - self._assetDependency.append(val) + AssetDependency = attr.ib(factory=partial(DeadlineIndexedVar, + "AssetDependency")) # Tile Job # ---------------------------------------------- @@ -402,7 +326,7 @@ class DeadlineJobInfo(object): """ def filter_data(a, v): - if a.name.startswith("_"): + if isinstance(v, DeadlineIndexedVar): return False if v is None: return False @@ -410,16 +334,37 @@ class DeadlineJobInfo(object): serialized = attr.asdict( self, dict_factory=OrderedDict, filter=filter_data) - serialized.update(self.EnvironmentKeyValue) - serialized.update(self.ExtraInfo) - serialized.update(self.ExtraInfoKeyValue) - serialized.update(self.TaskExtraInfoName) - serialized.update(self.OutputFilename) - serialized.update(self.OutputFilenameTile) - serialized.update(self.OutputDirectory) - serialized.update(self.AssetDependency) + + # Custom serialize these attributes + for attribute in [ + self.EnvironmentKeyValue, + self.ExtraInfo, + self.ExtraInfoKeyValue, + self.TaskExtraInfoName, + self.OutputFilename, + self.OutputFilenameTile, + self.OutputDirectory, + self.AssetDependency + ]: + serialized.update(attribute.serialize()) + return serialized + def update(self, data): + """Update instance with data dict""" + for key, value in data.items(): + setattr(self, key, value) + + def __setattr__(self, key, value): + # Backwards compatibility: Allow appending to index vars by setting + # it on Job Info directly like: JobInfo.OutputFilename = filename + existing = getattr(self, key, None) + if isinstance(existing, DeadlineIndexedVar): + existing += value + return + + object.__setattr__(self, key, value) + @six.add_metaclass(AbstractMetaInstancePlugin) class AbstractSubmitDeadline(pyblish.api.InstancePlugin): diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 2afa1883c4..d979c92814 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -110,8 +110,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): # Add options from RenderGlobals render_globals = instance.data.get("renderGlobals", {}) - for key, value in render_globals.items(): - setattr(job_info, key, value) + job_info.update(render_globals) keys = [ "FTRACK_API_KEY", @@ -257,8 +256,10 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): # As collected by super process() instance = self._instance - job_info = copy.deepcopy(self.job_info) - plugin_info = copy.deepcopy(self.plugin_info) + + payload_job_info, payload_plugin_info = payload + job_info = copy.deepcopy(payload_job_info) + plugin_info = copy.deepcopy(payload_plugin_info) # if we have sequence of files, we need to create tile job for # every frame @@ -309,16 +310,17 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): new_plugin_info = copy.deepcopy(plugin_info) # Add tile data into job info and plugin info - tiles_out, _ = _format_tiles( + tiles_data = _format_tiles( file, 0, instance.data.get("tilesX"), instance.data.get("tilesY"), instance.data.get("resolutionWidth"), instance.data.get("resolutionHeight"), - payload["PluginInfo"]["OutputFilePrefix"] - ) - new_job_info.update(tiles_out["JobInfo"]) - new_plugin_info.update(tiles_out["PluginInfo"]) + payload_plugin_info["OutputFilePrefix"] + )[0] + + new_job_info.update(tiles_data["JobInfo"]) + new_plugin_info.update(tiles_data["PluginInfo"]) self.log.info("hashing {} - {}".format(file_index, file)) job_hash = hashlib.sha256( @@ -342,15 +344,13 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): # Submit frame tile jobs frame_tile_job_id = {} for frame, tile_job_payload in frame_payloads.items(): - response = self.submit(tile_job_payload) - job_id = response.json()["_id"] + job_id = self.submit(tile_job_payload) frame_tile_job_id[frame] = job_id # Define assembly payloads assembly_job_info = copy.deepcopy(job_info) assembly_job_info.Plugin = self.tile_assembler_plugin - assembly_job_info.Name = "{job.Name} - Tile Assembly Job".format( - job=job_info) + assembly_job_info.Name += " - Tile Assembly Job" assembly_job_info.Frames = 1 assembly_job_info.MachineLimit = 1 assembly_job_info.Priority = instance.data.get("tile_priority", @@ -411,10 +411,9 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): instance.data.get("tilesY"), instance.data.get("resolutionWidth"), instance.data.get("resolutionHeight"), - payload["PluginInfo"]["OutputFilePrefix"] + payload_plugin_info["OutputFilePrefix"] )[1] - sorted(tiles) - for k, v in tiles.items(): + for k, v in sorted(tiles.items()): print("{}={}".format(k, v), file=cf) payload = self.assemble_payload( @@ -431,8 +430,8 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): self.log.info("submitting assembly job {} of {}".format( i+1, len(assembly_payloads) )) - response = self.submit(payload) - assembly_job_ids.append(response.json()["_id"]) + assembly_job_id = self.submit(payload) + assembly_job_ids.append(assembly_job_id) instance.data["assemblySubmissionJobs"] = assembly_job_ids From a7293f2a4f6a30297eea15297f5b25100e11e9f1 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 30 Aug 2022 14:13:14 +0200 Subject: [PATCH 059/122] Fix indentation --- openpype/modules/deadline/abstract_submit_deadline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py index 427faec115..e1bdcb10d9 100644 --- a/openpype/modules/deadline/abstract_submit_deadline.py +++ b/openpype/modules/deadline/abstract_submit_deadline.py @@ -295,7 +295,7 @@ class DeadlineJobInfo(object): OutputFilename = attr.ib(factory=partial(DeadlineIndexedVar, "OutputFilename")) OutputFilenameTile = attr.ib(factory=partial(DeadlineIndexedVar, - "OutputFilename{}Tile")) + "OutputFilename{}Tile")) OutputDirectory = attr.ib(factory=partial(DeadlineIndexedVar, "OutputDirectory")) From 91a3d8494bf8b65fe37560a02018edf59433caa6 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 30 Aug 2022 14:13:39 +0200 Subject: [PATCH 060/122] Disable aux files for now since it's not supported by Deadline Webservice --- .../modules/deadline/plugins/publish/submit_maya_deadline.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index d979c92814..7694e80e9a 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -419,8 +419,9 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): payload = self.assemble_payload( job_info=frame_assembly_job_info, plugin_info=assembly_plugin_info.copy(), + # todo: aux file transfers don't work with deadline webservice # add config file as job auxFile - aux_files=[config_file] + # aux_files=[config_file] ) assembly_payloads.append(payload) From 39d216797dacad00a3f42102e146c2806b9f8244 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 30 Aug 2022 14:17:40 +0200 Subject: [PATCH 061/122] Force integer pixel values --- .../deadline/plugins/publish/submit_maya_deadline.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 7694e80e9a..3fbff0153b 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -765,10 +765,10 @@ def _format_tiles( tiles_x, tiles_y ) - top = int(height) - (tile_y * h_space) - bottom = int(height) - ((tile_y - 1) * h_space) - 1 - left = (tile_x - 1) * w_space - right = (tile_x * w_space) - 1 + top = int(height - (tile_y * h_space)) + bottom = int(height - ((tile_y - 1) * h_space) - 1) + left = int((tile_x - 1) * w_space) + right = int((tile_x * w_space) - 1) # Job Info new_filename = "{}/{}{}".format( From 2fb7cabca49bae51162e2a82dfdc7d225094ed36 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 30 Aug 2022 14:39:28 +0200 Subject: [PATCH 062/122] Shush hound --- .../deadline/abstract_submit_deadline.py | 4 ++-- .../plugins/publish/submit_maya_deadline.py | 21 +++++++++++-------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py index e1bdcb10d9..35b114da95 100644 --- a/openpype/modules/deadline/abstract_submit_deadline.py +++ b/openpype/modules/deadline/abstract_submit_deadline.py @@ -623,8 +623,8 @@ class AbstractSubmitDeadline(pyblish.api.InstancePlugin): for i in context: is_workfile = ( - "workfile" in i.data.get("families", []) or - i.data["family"] == "workfile" + "workfile" in i.data.get("families", []) or + i.data["family"] == "workfile" ) if not is_workfile: continue diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 3fbff0153b..1b69f8b4e9 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -357,14 +357,14 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): self.tile_priority) assembly_plugin_info = { - "CleanupTiles": 1, - "ErrorOnMissing": True, - "Renderer": self._instance.data["renderer"] + "CleanupTiles": 1, + "ErrorOnMissing": True, + "Renderer": self._instance.data["renderer"] } assembly_payloads = [] output_dir = self.job_info.OutputDirectory[0] - for i, file in enumerate(assembly_files): + for file in assembly_files: frame = re.search(R_FRAME_NUMBER, file).group("frame") frame_assembly_job_info = copy.deepcopy(assembly_job_info) @@ -383,7 +383,8 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): # write assembly job config files now = datetime.now() - config_file = os.path.join(output_dir, + config_file = os.path.join( + output_dir, "{}_config_{}.txt".format( os.path.splitext(file)[0], now.strftime("%Y_%m_%d_%H_%M_%S") @@ -427,10 +428,12 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): # Submit assembly jobs assembly_job_ids = [] + num_assemblies = len(assembly_payloads) for i, payload in enumerate(assembly_payloads): - self.log.info("submitting assembly job {} of {}".format( - i+1, len(assembly_payloads) - )) + self.log.info( + "submitting assembly job {} of {}".format(i + 1, + num_assemblies) + ) assembly_job_id = self.submit(payload) assembly_job_ids.append(assembly_job_id) @@ -682,7 +685,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): pf.writelines(scene_data) pf.truncate() self.log.info("Applied {} patch to scene.".format( - patches[i]["name"] + patches[i]["name"] )) def _job_info_label(self, label): From e9e01e3163079bc9f6c48fe633aed92592928328 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 30 Aug 2022 14:43:18 +0200 Subject: [PATCH 063/122] Use update method --- .../modules/deadline/plugins/publish/submit_maya_deadline.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 1b69f8b4e9..9692b136e9 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -69,8 +69,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): # todo: test whether this works for existing production cases # where custom jobInfo was stored in the project settings - for key, value in self.jobInfo.items(): - setattr(job_info, key, value) + job_info.update(self.jobInfo) instance = self._instance context = instance.context From acf3d67f242625f8a91317b602ffa6467ff7cc6b Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 31 Aug 2022 15:59:20 +0200 Subject: [PATCH 064/122] moved 'create_workdir_extra_folders' to 'openpype.pipeline.workfile' --- .../hooks/pre_create_extra_workdir_folders.py | 6 +- openpype/lib/path_tools.py | 89 ++++++++++++------- openpype/lib/plugin_tools.py | 1 - openpype/pipeline/workfile/__init__.py | 4 + openpype/pipeline/workfile/path_resolving.py | 57 ++++++++++++ openpype/tools/workfiles/files_widget.py | 10 +-- 6 files changed, 126 insertions(+), 41 deletions(-) diff --git a/openpype/hooks/pre_create_extra_workdir_folders.py b/openpype/hooks/pre_create_extra_workdir_folders.py index d79c5831ee..c5af620c87 100644 --- a/openpype/hooks/pre_create_extra_workdir_folders.py +++ b/openpype/hooks/pre_create_extra_workdir_folders.py @@ -1,8 +1,6 @@ import os -from openpype.lib import ( - PreLaunchHook, - create_workdir_extra_folders -) +from openpype.lib import PreLaunchHook +from openpype.pipeline.workfile import create_workdir_extra_folders class AddLastWorkfileToLaunchArgs(PreLaunchHook): diff --git a/openpype/lib/path_tools.py b/openpype/lib/path_tools.py index f60b2fa722..671591bca4 100644 --- a/openpype/lib/path_tools.py +++ b/openpype/lib/path_tools.py @@ -16,6 +16,51 @@ from .profiles_filtering import filter_profiles log = logging.getLogger(__name__) +class PathToolsDeprecatedWarning(DeprecationWarning): + pass + + +def deprecated(new_destination): + """Mark functions as deprecated. + + It will result in a warning being emitted when the function is used. + """ + + func = None + if callable(new_destination): + func = new_destination + new_destination = None + + def _decorator(decorated_func): + if new_destination is None: + warning_message = ( + " Please check content of deprecated function to figure out" + " possible replacement." + ) + else: + warning_message = " Please replace your usage with '{}'.".format( + new_destination + ) + + @functools.wraps(decorated_func) + def wrapper(*args, **kwargs): + warnings.simplefilter("always", PathToolsDeprecatedWarning) + warnings.warn( + ( + "Call to deprecated function '{}'" + "\nFunction was moved or removed.{}" + ).format(decorated_func.__name__, warning_message), + category=PathToolsDeprecatedWarning, + stacklevel=4 + ) + return decorated_func(*args, **kwargs) + return wrapper + + if func is None: + return _decorator + return _decorator(func) + + def format_file_size(file_size, suffix=None): """Returns formatted string with size in appropriate unit. @@ -333,6 +378,7 @@ def get_project_basic_paths(project_name): return _list_path_items(folder_structure) +@deprecated("openpype.pipeline.workfile.create_workdir_extra_folders") def create_workdir_extra_folders( workdir, host_name, task_type, task_name, project_name, project_settings=None @@ -349,37 +395,18 @@ def create_workdir_extra_folders( project_name (str): Name of project on which task is. project_settings (dict): Prepared project settings. Are loaded if not passed. + + Deprecated: + Function will be removed after release version 3.16.* """ - # Load project settings if not set - if not project_settings: - project_settings = get_project_settings(project_name) - # Load extra folders profiles - extra_folders_profiles = ( - project_settings["global"]["tools"]["Workfiles"]["extra_folders"] + from openpype.pipeline.project_folders import create_workdir_extra_folders + + return create_workdir_extra_folders( + workdir, + host_name, + task_type, + task_name, + project_name, + project_settings ) - # Skip if are empty - if not extra_folders_profiles: - return - - # Prepare profiles filters - filter_data = { - "task_types": task_type, - "task_names": task_name, - "hosts": host_name - } - profile = filter_profiles(extra_folders_profiles, filter_data) - if profile is None: - return - - for subfolder in profile["folders"]: - # Make sure backslashes are converted to forwards slashes - # and does not start with slash - subfolder = subfolder.replace("\\", "/").lstrip("/") - # Skip empty strings - if not subfolder: - continue - - fullpath = os.path.join(workdir, subfolder) - if not os.path.exists(fullpath): - os.makedirs(fullpath) diff --git a/openpype/lib/plugin_tools.py b/openpype/lib/plugin_tools.py index 81d268ea1c..1e157dfbfd 100644 --- a/openpype/lib/plugin_tools.py +++ b/openpype/lib/plugin_tools.py @@ -3,7 +3,6 @@ import os import logging import re -import json import warnings import functools diff --git a/openpype/pipeline/workfile/__init__.py b/openpype/pipeline/workfile/__init__.py index 0aad29b6f9..94ecc81bd6 100644 --- a/openpype/pipeline/workfile/__init__.py +++ b/openpype/pipeline/workfile/__init__.py @@ -9,6 +9,8 @@ from .path_resolving import ( get_custom_workfile_template, get_custom_workfile_template_by_string_context, + + create_workdir_extra_folders, ) from .build_workfile import BuildWorkfile @@ -26,5 +28,7 @@ __all__ = ( "get_custom_workfile_template", "get_custom_workfile_template_by_string_context", + "create_workdir_extra_folders", + "BuildWorkfile", ) diff --git a/openpype/pipeline/workfile/path_resolving.py b/openpype/pipeline/workfile/path_resolving.py index 6d9e72dbd2..1243e84148 100644 --- a/openpype/pipeline/workfile/path_resolving.py +++ b/openpype/pipeline/workfile/path_resolving.py @@ -467,3 +467,60 @@ def get_custom_workfile_template_by_string_context( return get_custom_workfile_template( project_doc, asset_doc, task_name, host_name, anatomy, project_settings ) + + +def create_workdir_extra_folders( + workdir, + host_name, + task_type, + task_name, + project_name, + project_settings=None +): + """Create extra folders in work directory based on context. + + Args: + workdir (str): Path to workdir where workfiles is stored. + host_name (str): Name of host implementation. + task_type (str): Type of task for which extra folders should be + created. + task_name (str): Name of task for which extra folders should be + created. + project_name (str): Name of project on which task is. + project_settings (dict): Prepared project settings. Are loaded if not + passed. + """ + + # Load project settings if not set + if not project_settings: + project_settings = get_project_settings(project_name) + + # Load extra folders profiles + extra_folders_profiles = ( + project_settings["global"]["tools"]["Workfiles"]["extra_folders"] + ) + # Skip if are empty + if not extra_folders_profiles: + return + + # Prepare profiles filters + filter_data = { + "task_types": task_type, + "task_names": task_name, + "hosts": host_name + } + profile = filter_profiles(extra_folders_profiles, filter_data) + if profile is None: + return + + for subfolder in profile["folders"]: + # Make sure backslashes are converted to forwards slashes + # and does not start with slash + subfolder = subfolder.replace("\\", "/").lstrip("/") + # Skip empty strings + if not subfolder: + continue + + fullpath = os.path.join(workdir, subfolder) + if not os.path.exists(fullpath): + os.makedirs(fullpath) diff --git a/openpype/tools/workfiles/files_widget.py b/openpype/tools/workfiles/files_widget.py index a5d5b14bb6..b4f5e422bc 100644 --- a/openpype/tools/workfiles/files_widget.py +++ b/openpype/tools/workfiles/files_widget.py @@ -10,10 +10,7 @@ from openpype.host import IWorkfileHost from openpype.client import get_asset_by_id from openpype.tools.utils import PlaceholderLineEdit from openpype.tools.utils.delegates import PrettyTimeDelegate -from openpype.lib import ( - emit_event, - create_workdir_extra_folders, -) +from openpype.lib import emit_event from openpype.pipeline import ( registered_host, legacy_io, @@ -23,7 +20,10 @@ from openpype.pipeline.context_tools import ( compute_session_changes, change_current_context ) -from openpype.pipeline.workfile import get_workfile_template_key +from openpype.pipeline.workfile import ( + get_workfile_template_key, + create_workdir_extra_folders, +) from .model import ( WorkAreaFilesModel, From d3a9610c5a92f17bd319843eab4a99f6d68acce4 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 31 Aug 2022 16:09:15 +0200 Subject: [PATCH 065/122] moved helper function to pipeline.project_folders --- openpype/lib/path_tools.py | 64 +++++++++++----------------- openpype/pipeline/project_folders.py | 49 +++++++++++++++++++++ 2 files changed, 73 insertions(+), 40 deletions(-) create mode 100644 openpype/pipeline/project_folders.py diff --git a/openpype/lib/path_tools.py b/openpype/lib/path_tools.py index 671591bca4..736eb0effc 100644 --- a/openpype/lib/path_tools.py +++ b/openpype/lib/path_tools.py @@ -277,58 +277,42 @@ def get_last_version_from_path(path_dir, filter): return None +@deprecated("openpype.pipeline.project_folders.concatenate_splitted_paths") def concatenate_splitted_paths(split_paths, anatomy): - pattern_array = re.compile(r"\[.*\]") - output = [] - for path_items in split_paths: - clean_items = [] - if isinstance(path_items, str): - path_items = [path_items] + """ + Deprecated: + Function will be removed after release version 3.16.* + """ - for path_item in path_items: - if not re.match(r"{.+}", path_item): - path_item = re.sub(pattern_array, "", path_item) - clean_items.append(path_item) + from openpype.pipeline.project_folders import concatenate_splitted_paths - # backward compatibility - if "__project_root__" in path_items: - for root, root_path in anatomy.roots.items(): - if not os.path.exists(str(root_path)): - log.debug("Root {} path path {} not exist on \ - computer!".format(root, root_path)) - continue - clean_items = ["{{root[{}]}}".format(root), - r"{project[name]}"] + clean_items[1:] - output.append(os.path.normpath(os.path.sep.join(clean_items))) - continue - - output.append(os.path.normpath(os.path.sep.join(clean_items))) - - return output + return concatenate_splitted_paths(split_paths, anatomy) +@deprecated def get_format_data(anatomy): - project_doc = get_project(anatomy.project_name, fields=["data.code"]) - project_code = project_doc["data"]["code"] + """ + Deprecated: + Function will be removed after release version 3.16.* + """ - return { - "root": anatomy.roots, - "project": { - "name": anatomy.project_name, - "code": project_code - }, - } + from openpype.pipeline.template_data import get_project_template_data + + data = get_project_template_data(project_name=anatomy.project_name) + data["root"] = anatomy.roots + return data +@deprecated("openpype.pipeline.project_folders.fill_paths") def fill_paths(path_list, anatomy): - format_data = get_format_data(anatomy) - filled_paths = [] + """ + Deprecated: + Function will be removed after release version 3.16.* + """ - for path in path_list: - new_path = path.format(**format_data) - filled_paths.append(new_path) + from openpype.pipeline.project_folders import fill_paths - return filled_paths + return fill_paths(path_list, anatomy) def create_project_folders(basic_paths, project_name): diff --git a/openpype/pipeline/project_folders.py b/openpype/pipeline/project_folders.py new file mode 100644 index 0000000000..256c4e73d8 --- /dev/null +++ b/openpype/pipeline/project_folders.py @@ -0,0 +1,49 @@ +import os +import re + +from openpype.lib import Logger + +from .template_data import get_project_template_data + + +def concatenate_splitted_paths(split_paths, anatomy): + log = Logger.get_logger("concatenate_splitted_paths") + pattern_array = re.compile(r"\[.*\]") + output = [] + for path_items in split_paths: + clean_items = [] + if isinstance(path_items, str): + path_items = [path_items] + + for path_item in path_items: + if not re.match(r"{.+}", path_item): + path_item = re.sub(pattern_array, "", path_item) + clean_items.append(path_item) + + # backward compatibility + if "__project_root__" in path_items: + for root, root_path in anatomy.roots.items(): + if not os.path.exists(str(root_path)): + log.debug("Root {} path path {} not exist on \ + computer!".format(root, root_path)) + continue + clean_items = ["{{root[{}]}}".format(root), + r"{project[name]}"] + clean_items[1:] + output.append(os.path.normpath(os.path.sep.join(clean_items))) + continue + + output.append(os.path.normpath(os.path.sep.join(clean_items))) + + return output + + +def fill_paths(path_list, anatomy): + format_data = get_project_template_data(project_name=anatomy.project_name) + format_data["root"] = anatomy.roots + filled_paths = [] + + for path in path_list: + new_path = path.format(**format_data) + filled_paths.append(new_path) + + return filled_paths From c7ffda124e36813d00757d5096475bcb902c60f3 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 31 Aug 2022 16:10:54 +0200 Subject: [PATCH 066/122] moved 'create_project_folders' and 'get_project_basic_paths' --- openpype/lib/path_tools.py | 65 ++++++------------- .../action_create_project_structure.py | 7 +- openpype/pipeline/project_folders.py | 56 ++++++++++++++++ .../project_manager/project_manager/window.py | 24 +++---- 4 files changed, 88 insertions(+), 64 deletions(-) diff --git a/openpype/lib/path_tools.py b/openpype/lib/path_tools.py index 736eb0effc..5800498b07 100644 --- a/openpype/lib/path_tools.py +++ b/openpype/lib/path_tools.py @@ -1,21 +1,17 @@ import os import re -import abc -import json import logging -import six import platform +import functools +import warnings import clique -from openpype.client import get_project -from openpype.settings import get_project_settings - -from .profiles_filtering import filter_profiles - log = logging.getLogger(__name__) + + class PathToolsDeprecatedWarning(DeprecationWarning): pass @@ -315,51 +311,28 @@ def fill_paths(path_list, anatomy): return fill_paths(path_list, anatomy) +@deprecated("openpype.pipeline.project_folders.create_project_folders") def create_project_folders(basic_paths, project_name): - from openpype.pipeline import Anatomy - anatomy = Anatomy(project_name) + """ + Deprecated: + Function will be removed after release version 3.16.* + """ - concat_paths = concatenate_splitted_paths(basic_paths, anatomy) - filled_paths = fill_paths(concat_paths, anatomy) + from openpype.pipeline.project_folders import create_project_folders - # Create folders - for path in filled_paths: - if os.path.exists(path): - log.debug("Folder already exists: {}".format(path)) - else: - log.debug("Creating folder: {}".format(path)) - os.makedirs(path) - - -def _list_path_items(folder_structure): - output = [] - for key, value in folder_structure.items(): - if not value: - output.append(key) - else: - paths = _list_path_items(value) - for path in paths: - if not isinstance(path, (list, tuple)): - path = [path] - - item = [key] - item.extend(path) - output.append(item) - - return output + return create_project_folders(project_name, basic_paths) +@deprecated("openpype.pipeline.project_folders.get_project_basic_paths") def get_project_basic_paths(project_name): - project_settings = get_project_settings(project_name) - folder_structure = ( - project_settings["global"]["project_folder_structure"] - ) - if not folder_structure: - return [] + """ + Deprecated: + Function will be removed after release version 3.16.* + """ - if isinstance(folder_structure, str): - folder_structure = json.loads(folder_structure) - return _list_path_items(folder_structure) + from openpype.pipeline.project_folders import get_project_basic_paths + + return get_project_basic_paths(project_name) @deprecated("openpype.pipeline.workfile.create_workdir_extra_folders") diff --git a/openpype/modules/ftrack/event_handlers_user/action_create_project_structure.py b/openpype/modules/ftrack/event_handlers_user/action_create_project_structure.py index df914de854..7c896570b1 100644 --- a/openpype/modules/ftrack/event_handlers_user/action_create_project_structure.py +++ b/openpype/modules/ftrack/event_handlers_user/action_create_project_structure.py @@ -1,7 +1,10 @@ import re +from openpype.pipeline.project_folders import ( + get_project_basic_paths, + create_project_folders, +) from openpype_modules.ftrack.lib import BaseAction, statics_icon -from openpype.api import get_project_basic_paths, create_project_folders class CreateProjectFolders(BaseAction): @@ -81,7 +84,7 @@ class CreateProjectFolders(BaseAction): } # Invoking OpenPype API to create the project folders - create_project_folders(basic_paths, project_name) + create_project_folders(project_name, basic_paths) self.create_ftrack_entities(basic_paths, project_entity) self.trigger_event( diff --git a/openpype/pipeline/project_folders.py b/openpype/pipeline/project_folders.py index 256c4e73d8..811b9aa648 100644 --- a/openpype/pipeline/project_folders.py +++ b/openpype/pipeline/project_folders.py @@ -1,8 +1,13 @@ import os import re +import json +import six + +from openpype.settings import get_project_settings from openpype.lib import Logger +from .anatomy import Anatomy from .template_data import get_project_template_data @@ -47,3 +52,54 @@ def fill_paths(path_list, anatomy): filled_paths.append(new_path) return filled_paths + + +def create_project_folders(project_name, basic_paths=None): + log = Logger.get_logger("create_project_folders") + anatomy = Anatomy(project_name) + if basic_paths is None: + basic_paths = get_project_basic_paths(project_name) + + concat_paths = concatenate_splitted_paths(basic_paths, anatomy) + filled_paths = fill_paths(concat_paths, anatomy) + + # Create folders + for path in filled_paths: + if os.path.exists(path): + log.debug("Folder already exists: {}".format(path)) + else: + log.debug("Creating folder: {}".format(path)) + os.makedirs(path) + return filled_paths + + +def _list_path_items(folder_structure): + output = [] + for key, value in folder_structure.items(): + if not value: + output.append(key) + continue + + paths = _list_path_items(value) + for path in paths: + if not isinstance(path, (list, tuple)): + path = [path] + + item = [key] + item.extend(path) + output.append(item) + + return output + + +def get_project_basic_paths(project_name): + project_settings = get_project_settings(project_name) + folder_structure = ( + project_settings["global"]["project_folder_structure"] + ) + if not folder_structure: + return [] + + if isinstance(folder_structure, six.string_types): + folder_structure = json.loads(folder_structure) + return _list_path_items(folder_structure) diff --git a/openpype/tools/project_manager/project_manager/window.py b/openpype/tools/project_manager/project_manager/window.py index c6ae0ff352..3b2dea8ca3 100644 --- a/openpype/tools/project_manager/project_manager/window.py +++ b/openpype/tools/project_manager/project_manager/window.py @@ -1,5 +1,12 @@ from Qt import QtWidgets, QtCore, QtGui +from openpype import resources +from openpype.style import load_stylesheet +from openpype.widgets import PasswordDialog +from openpype.lib import is_admin_password_required, Logger +from openpype.pipeline import AvalonMongoDB +from openpype.pipeline.project_folders import create_project_folders + from . import ( ProjectModel, ProjectProxyFilter, @@ -13,17 +20,6 @@ from . import ( ) from .widgets import ConfirmProjectDeletion from .style import ResourceCache -from openpype.style import load_stylesheet -from openpype.lib import is_admin_password_required -from openpype.widgets import PasswordDialog -from openpype.pipeline import AvalonMongoDB - -from openpype import resources -from openpype.api import ( - get_project_basic_paths, - create_project_folders, - Logger -) class ProjectManagerWindow(QtWidgets.QWidget): @@ -259,12 +255,8 @@ class ProjectManagerWindow(QtWidgets.QWidget): qm.Yes | qm.No) if ans == qm.Yes: try: - # Get paths based on presets - basic_paths = get_project_basic_paths(project_name) - if not basic_paths: - pass # Invoking OpenPype API to create the project folders - create_project_folders(basic_paths, project_name) + create_project_folders(project_name) except Exception as exc: self.log.warning( "Cannot create starting folders: {}".format(exc), From 27e74ee0095d33c22ee2059f12efb508662d066f Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 31 Aug 2022 16:15:28 +0200 Subject: [PATCH 067/122] skip folders creation if are not set --- openpype/pipeline/project_folders.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/openpype/pipeline/project_folders.py b/openpype/pipeline/project_folders.py index 811b9aa648..1bcba5c320 100644 --- a/openpype/pipeline/project_folders.py +++ b/openpype/pipeline/project_folders.py @@ -60,6 +60,9 @@ def create_project_folders(project_name, basic_paths=None): if basic_paths is None: basic_paths = get_project_basic_paths(project_name) + if not basic_paths: + return + concat_paths = concatenate_splitted_paths(basic_paths, anatomy) filled_paths = fill_paths(concat_paths, anatomy) @@ -70,7 +73,6 @@ def create_project_folders(project_name, basic_paths=None): else: log.debug("Creating folder: {}".format(path)) os.makedirs(path) - return filled_paths def _list_path_items(folder_structure): From 111a0be1a858f7568a169196c0be32e5d9da4a88 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 31 Aug 2022 16:21:23 +0200 Subject: [PATCH 068/122] removed not needed lines --- openpype/lib/path_tools.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/openpype/lib/path_tools.py b/openpype/lib/path_tools.py index 5800498b07..0b6d0a3391 100644 --- a/openpype/lib/path_tools.py +++ b/openpype/lib/path_tools.py @@ -10,8 +10,6 @@ import clique log = logging.getLogger(__name__) - - class PathToolsDeprecatedWarning(DeprecationWarning): pass From 1f06830a03fa58ccfa2ccf76f940ec0281343c9c Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 1 Sep 2022 13:24:20 +0200 Subject: [PATCH 069/122] change both 'user' and 'username' keys --- openpype/plugins/publish/extract_burnin.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openpype/plugins/publish/extract_burnin.py b/openpype/plugins/publish/extract_burnin.py index 88093fb92f..36a28beb5d 100644 --- a/openpype/plugins/publish/extract_burnin.py +++ b/openpype/plugins/publish/extract_burnin.py @@ -492,6 +492,7 @@ class ExtractBurnin(openpype.api.Extractor): # OPENPYPE_USERNAME might have side effects webpublish_user_name = os.environ.get("WEBPUBLISH_OPENPYPE_USERNAME") if webpublish_user_name: + burnin_data["user"] = webpublish_user_name burnin_data["username"] = webpublish_user_name self.log.debug( From 398325684961bf9b408da0e56011cfbd4eb8bed1 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 1 Sep 2022 13:24:33 +0200 Subject: [PATCH 070/122] add 'user' to representation context data --- openpype/plugins/publish/integrate.py | 2 +- openpype/plugins/publish/integrate_hero_version.py | 2 +- openpype/plugins/publish/integrate_legacy.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/openpype/plugins/publish/integrate.py b/openpype/plugins/publish/integrate.py index f99c718f8a..788966878f 100644 --- a/openpype/plugins/publish/integrate.py +++ b/openpype/plugins/publish/integrate.py @@ -135,7 +135,7 @@ class IntegrateAsset(pyblish.api.InstancePlugin): # the database even if not used by the destination template db_representation_context_keys = [ "project", "asset", "task", "subset", "version", "representation", - "family", "hierarchy", "username", "output" + "family", "hierarchy", "username", "user", "output" ] skip_host_families = [] diff --git a/openpype/plugins/publish/integrate_hero_version.py b/openpype/plugins/publish/integrate_hero_version.py index 7d698ff98d..5b8b141e88 100644 --- a/openpype/plugins/publish/integrate_hero_version.py +++ b/openpype/plugins/publish/integrate_hero_version.py @@ -46,7 +46,7 @@ class IntegrateHeroVersion(pyblish.api.InstancePlugin): ignored_representation_names = [] db_representation_context_keys = [ "project", "asset", "task", "subset", "representation", - "family", "hierarchy", "task", "username" + "family", "hierarchy", "task", "username", "user" ] # QUESTION/TODO this process should happen on server if crashed due to # permissions error on files (files were used or user didn't have perms) diff --git a/openpype/plugins/publish/integrate_legacy.py b/openpype/plugins/publish/integrate_legacy.py index b90b61f587..5a6190f38e 100644 --- a/openpype/plugins/publish/integrate_legacy.py +++ b/openpype/plugins/publish/integrate_legacy.py @@ -127,7 +127,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): exclude_families = ["render.farm"] db_representation_context_keys = [ "project", "asset", "task", "subset", "version", "representation", - "family", "hierarchy", "task", "username" + "family", "hierarchy", "task", "username", "user" ] default_template_name = "publish" From 198ad3ac901dc962ab840a0b09a5bd25d4a286e2 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 1 Sep 2022 13:25:02 +0200 Subject: [PATCH 071/122] add both 'user' and 'username' in integrate slack --- .../modules/slack/plugins/publish/integrate_slack_api.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/openpype/modules/slack/plugins/publish/integrate_slack_api.py b/openpype/modules/slack/plugins/publish/integrate_slack_api.py index c3b288f0cd..4a8e9f773f 100644 --- a/openpype/modules/slack/plugins/publish/integrate_slack_api.py +++ b/openpype/modules/slack/plugins/publish/integrate_slack_api.py @@ -95,13 +95,15 @@ class IntegrateSlackAPI(pyblish.api.InstancePlugin): Reviews might be large, so allow only adding link to message instead of uploading only. """ + fill_data = copy.deepcopy(instance.context.data["anatomyData"]) + username = fill_data.get("user") fill_pairs = [ ("asset", instance.data.get("asset", fill_data.get("asset"))), ("subset", instance.data.get("subset", fill_data.get("subset"))), - ("username", instance.data.get("username", - fill_data.get("username"))), + ("user", username), + ("username", username), ("app", instance.data.get("app", fill_data.get("app"))), ("family", instance.data.get("family", fill_data.get("family"))), ("version", str(instance.data.get("version", From 6bcbf34d27e3806ae7f62a09d7d5a6002671a2eb Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 1 Sep 2022 13:53:39 +0200 Subject: [PATCH 072/122] changed class name to 'CollectUsernameForWebpublish' --- openpype/modules/ftrack/plugins/publish/collect_username.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/openpype/modules/ftrack/plugins/publish/collect_username.py b/openpype/modules/ftrack/plugins/publish/collect_username.py index a9b746ea51..0e232bf83e 100644 --- a/openpype/modules/ftrack/plugins/publish/collect_username.py +++ b/openpype/modules/ftrack/plugins/publish/collect_username.py @@ -13,7 +13,7 @@ import os import pyblish.api -class CollectUsername(pyblish.api.ContextPlugin): +class CollectUsernameForWebpublish(pyblish.api.ContextPlugin): """ Translates user email to Ftrack username. @@ -32,10 +32,8 @@ class CollectUsername(pyblish.api.ContextPlugin): hosts = ["webpublisher", "photoshop"] targets = ["remotepublish", "filespublish", "tvpaint_worker"] - _context = None - def process(self, context): - self.log.info("CollectUsername") + self.log.info("{}".format(self.__class__.__name__)) os.environ["FTRACK_API_USER"] = os.environ["FTRACK_BOT_API_USER"] os.environ["FTRACK_API_KEY"] = os.environ["FTRACK_BOT_API_KEY"] From 8e6b0567c967008b0b64151eeceed23a426ed073 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 1 Sep 2022 13:54:29 +0200 Subject: [PATCH 073/122] skip 'WEBPUBLISH_OPENPYPE_USERNAME' usage --- .../modules/ftrack/plugins/publish/collect_username.py | 1 - openpype/plugins/publish/extract_burnin.py | 7 ------- 2 files changed, 8 deletions(-) diff --git a/openpype/modules/ftrack/plugins/publish/collect_username.py b/openpype/modules/ftrack/plugins/publish/collect_username.py index 0e232bf83e..ab1f7d8d5d 100644 --- a/openpype/modules/ftrack/plugins/publish/collect_username.py +++ b/openpype/modules/ftrack/plugins/publish/collect_username.py @@ -65,5 +65,4 @@ class CollectUsernameForWebpublish(pyblish.api.ContextPlugin): burnin_name = username if '@' in burnin_name: burnin_name = burnin_name[:burnin_name.index('@')] - os.environ["WEBPUBLISH_OPENPYPE_USERNAME"] = burnin_name context.data["user"] = burnin_name diff --git a/openpype/plugins/publish/extract_burnin.py b/openpype/plugins/publish/extract_burnin.py index 36a28beb5d..c8b2b73874 100644 --- a/openpype/plugins/publish/extract_burnin.py +++ b/openpype/plugins/publish/extract_burnin.py @@ -488,13 +488,6 @@ class ExtractBurnin(openpype.api.Extractor): "frame_end_handle": frame_end_handle } - # use explicit username for webpublishes as rewriting - # OPENPYPE_USERNAME might have side effects - webpublish_user_name = os.environ.get("WEBPUBLISH_OPENPYPE_USERNAME") - if webpublish_user_name: - burnin_data["user"] = webpublish_user_name - burnin_data["username"] = webpublish_user_name - self.log.debug( "Basic burnin_data: {}".format(json.dumps(burnin_data, indent=4)) ) From 3ac5305f65b17083b126eeb76675064311bf4f4a Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 1 Sep 2022 13:54:58 +0200 Subject: [PATCH 074/122] better user query --- .../modules/ftrack/plugins/publish/collect_username.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/openpype/modules/ftrack/plugins/publish/collect_username.py b/openpype/modules/ftrack/plugins/publish/collect_username.py index ab1f7d8d5d..798f3960a8 100644 --- a/openpype/modules/ftrack/plugins/publish/collect_username.py +++ b/openpype/modules/ftrack/plugins/publish/collect_username.py @@ -1,5 +1,8 @@ """Loads publishing context from json and continues in publish process. +Should run before 'CollectAnatomyContextData' so the user on context is +changed before it's stored to context anatomy data or instance anatomy data. + Requires: anatomy -> context["anatomy"] *(pyblish.api.CollectorOrder - 0.11) @@ -52,12 +55,14 @@ class CollectUsernameForWebpublish(pyblish.api.ContextPlugin): return session = ftrack_api.Session(auto_connect_event_hub=False) - user = session.query("User where email like '{}'".format(user_email)) + user = session.query( + "User where email like '{}'".format(user_email) + ).first() if not user: raise ValueError( "Couldn't find user with {} email".format(user_email)) - user = user[0] + username = user.get("username") self.log.debug("Resolved ftrack username:: {}".format(username)) os.environ["FTRACK_API_USER"] = username From 3c6b999d544d65a8d31cc37ae54c731ff098f73a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Albert?= Date: Thu, 1 Sep 2022 17:22:14 +0200 Subject: [PATCH 075/122] Kitsu - sync_all_project - add list ignore_projects --- openpype/modules/kitsu/utils/update_op_with_zou.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/openpype/modules/kitsu/utils/update_op_with_zou.py b/openpype/modules/kitsu/utils/update_op_with_zou.py index 8d65591c0b..f013251bb1 100644 --- a/openpype/modules/kitsu/utils/update_op_with_zou.py +++ b/openpype/modules/kitsu/utils/update_op_with_zou.py @@ -318,13 +318,13 @@ def write_project_to_op(project: dict, dbcon: AvalonMongoDB) -> UpdateOne: ) -def sync_all_projects(login: str, password: str): +def sync_all_projects(login: str, password: str, ignore_projects: list = []): """Update all OP projects in DB with Zou data. Args: login (str): Kitsu user login password (str): Kitsu user password - + ignore_projects (list): List of unsynced project names Raises: gazu.exception.AuthFailedException: Wrong user login and/or password """ @@ -340,7 +340,8 @@ def sync_all_projects(login: str, password: str): dbcon.install() all_projects = gazu.project.all_open_projects() for project in all_projects: - sync_project_from_kitsu(dbcon, project) + if project["name"] not in ignore_projects: + sync_project_from_kitsu(dbcon, project) def sync_project_from_kitsu(dbcon: AvalonMongoDB, project: dict): From 4bdd18cb817bbc58c1143e0e02442a9346ce9a1e Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Fri, 2 Sep 2022 11:50:14 +0200 Subject: [PATCH 076/122] Use DeadlineKeyValueVar for EnvironmentKeyValue on Job Info - To improve readability of code that sets the values --- .../deadline/abstract_submit_deadline.py | 58 +++++++++++++++---- .../publish/submit_aftereffects_deadline.py | 11 ++-- .../publish/submit_harmony_deadline.py | 10 ++-- .../plugins/publish/submit_maya_deadline.py | 7 +-- 4 files changed, 59 insertions(+), 27 deletions(-) diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py index 35b114da95..beb1cd0fae 100644 --- a/openpype/modules/deadline/abstract_submit_deadline.py +++ b/openpype/modules/deadline/abstract_submit_deadline.py @@ -67,6 +67,43 @@ def requests_get(*args, **kwargs): return requests.get(*args, **kwargs) +class DeadlineKeyValueVar(dict): + """ + + Serializes dictionary key values as "{key}={value}" like Deadline uses + for EnvironmentKeyValue. + + As an example: + EnvironmentKeyValue0="A_KEY=VALUE_A" + EnvironmentKeyValue1="OTHER_KEY=VALUE_B" + + The keys are serialized in alphabetical order (sorted). + + Example: + >>> var = DeadlineKeyValueVar("EnvironmentKeyValue") + >>> var["my_var"] = "hello" + >>> var["my_other_var"] = "hello2" + >>> var.serialize() + + + """ + def __init__(self, key): + super(DeadlineKeyValueVar, self).__init__() + self.__key = key + + def serialize(self): + key = self.__key + + # Allow custom location for index in serialized string + if "{}" not in key: + key = key + "{}" + + return { + key.format(index): "{}={}".format(var_key, var_value) + for index, (var_key, var_value) in enumerate(sorted(self.items())) + } + + class DeadlineIndexedVar(dict): """ @@ -80,15 +117,9 @@ class DeadlineIndexedVar(dict): """ def __init__(self, key): + super(DeadlineIndexedVar, self).__init__() self.__key = key - def next_available_index(self): - # Add as first unused entry - i = 0 - while i in self.keys(): - i += 1 - return i - def serialize(self): key = self.__key @@ -100,6 +131,13 @@ class DeadlineIndexedVar(dict): key.format(index): value for index, value in sorted(self.items()) } + def next_available_index(self): + # Add as first unused entry + i = 0 + while i in self.keys(): + i += 1 + return i + def update(self, data): # Force the integer key check for key, value in data.items(): @@ -271,7 +309,7 @@ class DeadlineJobInfo(object): # Environment # ---------------------------------------------- - EnvironmentKeyValue = attr.ib(factory=partial(DeadlineIndexedVar, + EnvironmentKeyValue = attr.ib(factory=partial(DeadlineKeyValueVar, "EnvironmentKeyValue")) IncludeEnvironment = attr.ib(default=None) # Default: false @@ -281,7 +319,7 @@ class DeadlineJobInfo(object): # Job Extra Info # ---------------------------------------------- ExtraInfo = attr.ib(factory=partial(DeadlineIndexedVar, "ExtraInfo")) - ExtraInfoKeyValue = attr.ib(factory=partial(DeadlineIndexedVar, + ExtraInfoKeyValue = attr.ib(factory=partial(DeadlineKeyValueVar, "ExtraInfoKeyValue")) # Task Extra Info Names @@ -326,7 +364,7 @@ class DeadlineJobInfo(object): """ def filter_data(a, v): - if isinstance(v, DeadlineIndexedVar): + if isinstance(v, (DeadlineIndexedVar, DeadlineKeyValueVar)): return False if v is None: return False diff --git a/openpype/modules/deadline/plugins/publish/submit_aftereffects_deadline.py b/openpype/modules/deadline/plugins/publish/submit_aftereffects_deadline.py index 1d68793d53..55acd92043 100644 --- a/openpype/modules/deadline/plugins/publish/submit_aftereffects_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_aftereffects_deadline.py @@ -92,13 +92,12 @@ class AfterEffectsSubmitDeadline( environment = dict({key: os.environ[key] for key in keys if key in os.environ}, **legacy_io.Session) for key in keys: - val = environment.get(key) - if val: - dln_job_info.EnvironmentKeyValue = "{key}={value}".format( - key=key, - value=val) + value = environment.get(key) + if value: + dln_job_info.EnvironmentKeyValue[key] = value + # to recognize job from PYPE for turning Event On/Off - dln_job_info.EnvironmentKeyValue = "OPENPYPE_RENDER_JOB=1" + dln_job_info.EnvironmentKeyValue["OPENPYPE_RENDER_JOB"] = "1" return dln_job_info diff --git a/openpype/modules/deadline/plugins/publish/submit_harmony_deadline.py b/openpype/modules/deadline/plugins/publish/submit_harmony_deadline.py index 3f9c09b592..6327143623 100644 --- a/openpype/modules/deadline/plugins/publish/submit_harmony_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_harmony_deadline.py @@ -284,14 +284,12 @@ class HarmonySubmitDeadline( environment = dict({key: os.environ[key] for key in keys if key in os.environ}, **legacy_io.Session) for key in keys: - val = environment.get(key) - if val: - job_info.EnvironmentKeyValue = "{key}={value}".format( - key=key, - value=val) + value = environment.get(key) + if value: + job_info.EnvironmentKeyValue[key] = value # to recognize job from PYPE for turning Event On/Off - job_info.EnvironmentKeyValue = "OPENPYPE_RENDER_JOB=1" + job_info.EnvironmentKeyValue["OPENPYPE_RENDER_JOB"] = "1" return job_info diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 9692b136e9..ad46feea03 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -137,8 +137,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): for key, value in environment.items(): if not value: continue - job_info.EnvironmentKeyValue = "{key}={value}".format(key=key, - value=value) + job_info.EnvironmentKeyValue[key] = value # Adding file dependencies. if self.asset_dependencies: @@ -538,9 +537,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): for key, value in envs.items(): if not value: continue - - job_info.EnvironmentKeyValue = "{key}={value}".format(key=key, - value=value) + job_info.EnvironmentKeyValue[key] = value plugin_info.update({ "Version": "3.6", From 1e87c9d6d2c7338f8e53e8a06d9f1983056797b6 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Fri, 2 Sep 2022 12:02:39 +0200 Subject: [PATCH 077/122] Use DeadlineIndexedVar `__iadd__` functionality --- .../plugins/publish/submit_aftereffects_deadline.py | 4 ++-- .../deadline/plugins/publish/submit_maya_deadline.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_aftereffects_deadline.py b/openpype/modules/deadline/plugins/publish/submit_aftereffects_deadline.py index 55acd92043..0c1ffa6bd7 100644 --- a/openpype/modules/deadline/plugins/publish/submit_aftereffects_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_aftereffects_deadline.py @@ -67,9 +67,9 @@ class AfterEffectsSubmitDeadline( dln_job_info.Group = self.group dln_job_info.Department = self.department dln_job_info.ChunkSize = self.chunk_size - dln_job_info.OutputFilename = \ + dln_job_info.OutputFilename += \ os.path.basename(self._instance.data["expectedFiles"][0]) - dln_job_info.OutputDirectory = \ + dln_job_info.OutputDirectory += \ os.path.dirname(self._instance.data["expectedFiles"][0]) dln_job_info.JobDelay = "00:00:00" diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index ad46feea03..6b08f9894d 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -144,14 +144,14 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): dependencies = instance.context.data["fileDependencies"] dependencies.append(context.data["currentFile"]) for dependency in dependencies: - job_info.AssetDependency = dependency + job_info.AssetDependency += dependency # Add list of expected files to job # --------------------------------- exp = instance.data.get("expectedFiles") for filepath in self._iter_expected_files(exp): - job_info.OutputDirectory = os.path.dirname(filepath) - job_info.OutputFilename = os.path.basename(filepath) + job_info.OutputDirectory += os.path.dirname(filepath) + job_info.OutputFilename += os.path.basename(filepath) return job_info @@ -443,7 +443,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): if self.asset_dependencies: # Asset dependency to wait for at least the scene file to sync. - job_info.AssetDependency = self.scene_path + job_info.AssetDependency += self.scene_path # Get layer prefix render_products = self._instance.data["renderProducts"] From 2c01cb806d68aa04a733e16d4cfd1abb15f438fe Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Fri, 2 Sep 2022 12:03:34 +0200 Subject: [PATCH 078/122] Remove backwards compatibility for append functionality in old style vars --- openpype/modules/deadline/abstract_submit_deadline.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py index beb1cd0fae..f698b7688e 100644 --- a/openpype/modules/deadline/abstract_submit_deadline.py +++ b/openpype/modules/deadline/abstract_submit_deadline.py @@ -393,16 +393,6 @@ class DeadlineJobInfo(object): for key, value in data.items(): setattr(self, key, value) - def __setattr__(self, key, value): - # Backwards compatibility: Allow appending to index vars by setting - # it on Job Info directly like: JobInfo.OutputFilename = filename - existing = getattr(self, key, None) - if isinstance(existing, DeadlineIndexedVar): - existing += value - return - - object.__setattr__(self, key, value) - @six.add_metaclass(AbstractMetaInstancePlugin) class AbstractSubmitDeadline(pyblish.api.InstancePlugin): From 47164b36effa0f4986ccf15aa3a4967ccb014e26 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Fri, 2 Sep 2022 12:11:35 +0200 Subject: [PATCH 079/122] Be more explicit about what keys to include from Session This way it matches more with logic of other host submitters (e.g. AfterEffects + Harmony) --- .../deadline/plugins/publish/submit_maya_deadline.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 6b08f9894d..bb48fe6902 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -130,15 +130,16 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): environment = dict({key: os.environ[key] for key in keys if key in os.environ}, **legacy_io.Session) - # to recognize job from PYPE for turning Event On/Off - environment["OPENPYPE_RENDER_JOB"] = "1" - environment["OPENPYPE_LOG_NO_COLORS"] = "1" - - for key, value in environment.items(): + for key in keys: + value = environment.get(key) if not value: continue job_info.EnvironmentKeyValue[key] = value + # to recognize job from PYPE for turning Event On/Off + job_info.EnvironmentKeyValue["OPENPYPE_RENDER_JOB"] = "1" + job_info.EnvironmentKeyValue["OPENPYPE_LOG_NO_COLORS"] = "1" + # Adding file dependencies. if self.asset_dependencies: dependencies = instance.context.data["fileDependencies"] From 37371936cf293200b74b7d5ee5381aac8a0551ba Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 2 Sep 2022 14:08:40 +0200 Subject: [PATCH 080/122] safer task data --- .../plugins/publish/integrate_slack_api.py | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/openpype/modules/slack/plugins/publish/integrate_slack_api.py b/openpype/modules/slack/plugins/publish/integrate_slack_api.py index 4a8e9f773f..643e55915b 100644 --- a/openpype/modules/slack/plugins/publish/integrate_slack_api.py +++ b/openpype/modules/slack/plugins/publish/integrate_slack_api.py @@ -112,13 +112,19 @@ class IntegrateSlackAPI(pyblish.api.InstancePlugin): if review_path: fill_pairs.append(("review_filepath", review_path)) - task_data = instance.data.get("task") - if not task_data: - task_data = fill_data.get("task") - for key, value in task_data.items(): - fill_key = "task[{}]".format(key) - fill_pairs.append((fill_key, value)) - fill_pairs.append(("task", task_data["name"])) + task_data = fill_data.get("task") + if task_data: + if ( + "{task}" in message_templ + or "{Task}" in message_templ + or "{TASK}" in message_templ + ): + fill_pairs.append(("task", task_data["name"])) + + else: + for key, value in task_data.items(): + fill_key = "task[{}]".format(key) + fill_pairs.append((fill_key, value)) self.log.debug("fill_pairs ::{}".format(fill_pairs)) multiple_case_variants = prepare_template_data(fill_pairs) From 58fd5a1b097bd7d7e71f004b3fff8529296b1102 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Albert?= Date: Fri, 2 Sep 2022 16:32:52 +0200 Subject: [PATCH 081/122] Make to Optional Arguments --- openpype/modules/kitsu/utils/update_op_with_zou.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/kitsu/utils/update_op_with_zou.py b/openpype/modules/kitsu/utils/update_op_with_zou.py index f013251bb1..199c59053b 100644 --- a/openpype/modules/kitsu/utils/update_op_with_zou.py +++ b/openpype/modules/kitsu/utils/update_op_with_zou.py @@ -318,7 +318,7 @@ def write_project_to_op(project: dict, dbcon: AvalonMongoDB) -> UpdateOne: ) -def sync_all_projects(login: str, password: str, ignore_projects: list = []): +def sync_all_projects(login: str, password: str, ignore_projects=[]): """Update all OP projects in DB with Zou data. Args: From 88e4798b535c47cefce3dc2a1ed9aacf60dd0f68 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Fri, 2 Sep 2022 19:50:56 +0200 Subject: [PATCH 082/122] Remove old type hint --- .../modules/deadline/plugins/publish/submit_maya_deadline.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index bb48fe6902..68d55fef5d 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -641,7 +641,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): return result def _patch_workfile(self): - # type: (str, dict) -> [str, None] """Patch Maya scene. This will take list of patches (lines to add) and apply them to From 5645bcb353b13b1711ba67e0a3b394b273e7cef3 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Sat, 3 Sep 2022 13:17:10 +0200 Subject: [PATCH 083/122] Use custom plugin info per type of plugin submission --- .../plugins/publish/submit_maya_deadline.py | 127 +++++++++--------- 1 file changed, 66 insertions(+), 61 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 68d55fef5d..2a41d92efd 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -49,6 +49,30 @@ class MayaPluginInfo: RenderSetupIncludeLights = attr.ib(default=None) # Include all lights flag +@attr.s +class PythonPluginInfo: + ScriptFile = attr.ib() + Version = attr.ib(default="3.6") + Arguments = attr.ib(default=None) + SingleFrameOnly = attr.ib(default=None) + + +@attr.s +class VRayPluginInfo: + InputFilename = attr.ib(default=None) # Input + SeparateFilesPerFrame = attr.ib(default=None) + VRayEngine = attr.ib(default="V-Ray") + Width = attr.ib(default=None) + Height = attr.ib(default=None) # Mandatory for Deadline + OutputFilePath = attr.ib(default=True) + OutputFileName = attr.ib(default=None) # Render only this layer + + +@attr.s +class ArnoldPluginInfo: + ArnoldFile = attr.ib(default=None) + + class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): label = "Submit Render to Deadline" @@ -479,26 +503,19 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): def _get_vray_export_payload(self, data): job_info = copy.deepcopy(self.job_info) - job_info.Name = self._job_info_label("Export") # Get V-Ray settings info to compute output path - vray_settings = cmds.ls(type="VRaySettingsNode") - node = vray_settings[0] - template = cmds.getAttr("{}.vrscene_filename".format(node)) - scene, _ = os.path.splitext(data["filename"]) - first_file = self.format_vray_output_filename(scene, template) - first_file = "{}/{}".format(data["workspace"], first_file) - output = os.path.dirname(first_file) + vray_scene = self.format_vray_output_filename() plugin_info = { "Renderer": "vray", "SkipExistingFrames": True, "UseLegacyRenderLayers": True, - "OutputFilePath": output + "OutputFilePath": os.path.dirname(vray_scene) } - return job_info, plugin_info + return job_info, attr.asdict(plugin_info) def _get_arnold_export_payload(self, data): @@ -515,8 +532,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): script = os.path.normpath(module_path) job_info = copy.deepcopy(self.job_info) - plugin_info = copy.deepcopy(self.plugin_info) - job_info.Name = self._job_info_label("Export") # Force a single frame Python job @@ -540,14 +555,14 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): continue job_info.EnvironmentKeyValue[key] = value - plugin_info.update({ - "Version": "3.6", - "ScriptFile": script, - "Arguments": "", - "SingleFrameOnly": "True", - }) + plugin_info = PythonPluginInfo( + ScriptFile=script, + Version="3.6", + Arguments="", + SingleFrameOnly="True" + ) - return job_info, plugin_info + return job_info, attr.asdict(plugin_info) def _get_vray_render_payload(self, data): @@ -558,27 +573,17 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): job_info.OverrideTaskExtraInfoNames = False # Plugin Info - vray_settings = cmds.ls(type="VRaySettingsNode") - node = vray_settings[0] - template = cmds.getAttr("{}.vrscene_filename".format(node)) - # "vrayscene//_/" + plugin_info = VRayPluginInfo( + InputFilename=self.format_vray_output_filename(), + SeparateFilesPerFrame=False, + VRayEngine="V-Ray", + Width=self._instance.data["resolutionWidth"], + Height=self._instance.data["resolutionHeight"], + OutputFilePath=job_info.OutputDirectory[0], + OutputFileName=job_info.OutputFilename[0] + ) - scene, _ = os.path.splitext(self.scene_path) - first_file = self.format_vray_output_filename(scene, template) - first_file = "{}/{}".format(data["workspace"], first_file) - - plugin_info = { - "InputFilename": first_file, - "SeparateFilesPerFrame": True, - "VRayEngine": "V-Ray", - - "Width": self._instance.data["resolutionWidth"], - "Height": self._instance.data["resolutionHeight"], - "OutputFilePath": job_info.OutputDirectory[0], - "OutputFileName": job_info.OutputFilename[0] - } - - return job_info, plugin_info + return job_info, attr.asdict(plugin_info) def _get_arnold_render_payload(self, data): @@ -590,55 +595,55 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): # Plugin Info ass_file, _ = os.path.splitext(data["output_filename_0"]) - first_file = ass_file + ".ass" - plugin_info = { - "ArnoldFile": first_file, - } + ass_filepath = ass_file + ".ass" - return job_info, plugin_info + plugin_info = ArnoldPluginInfo( + ArnoldFile=ass_filepath + ) - def format_vray_output_filename(self, filename, template, dir=False): + return job_info, attr.asdict(plugin_info) + + def format_vray_output_filename(self): """Format the expected output file of the Export job. Example: /_/ - "shot010_v006/shot010_v006_CHARS/CHARS" - - Args: - instance: - filename(str): - dir(bool): - + "shot010_v006/shot010_v006_CHARS/CHARS_0001.vrscene" Returns: str """ + + # "vrayscene//_/" + vray_settings = cmds.ls(type="VRaySettingsNode") + node = vray_settings[0] + template = cmds.getAttr("{}.vrscene_filename".format(node)) + scene, _ = os.path.splitext(self.scene_path) + def smart_replace(string, key_values): new_string = string for key, value in key_values.items(): new_string = new_string.replace(key, value) return new_string - # Ensure filename has no extension - file_name, _ = os.path.splitext(filename) + # Get workfile scene path without extension to format vrscene_filename + scene_filename = os.path.basename(self.scene_path) + scene_filename_no_ext, _ = os.path.splitext(scene_filename) layer = self._instance.data['setMembers'] # Reformat without tokens output_path = smart_replace( template, - {"": file_name, + {"": scene_filename_no_ext, "": layer}) - if dir: - return output_path.replace("\\", "/") - start_frame = int(self._instance.data["frameStartHandle"]) + workspace = self._instance.context.data["workspace"] filename_zero = "{}_{:04d}.vrscene".format(output_path, start_frame) + filepath_zero = os.path.join(workspace, filename_zero) - result = filename_zero.replace("\\", "/") - - return result + return filepath_zero.replace("\\", "/") def _patch_workfile(self): """Patch Maya scene. From 507dac4aa9f50e8978a841067262ce33e77cf5e0 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Sat, 3 Sep 2022 15:06:24 +0200 Subject: [PATCH 084/122] Ensure integer math for _format_tiles See #3758 --- .../plugins/publish/submit_maya_deadline.py | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 2a41d92efd..7c486b7c34 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -754,14 +754,21 @@ def _format_tiles( used for assembler configuration. """ - tile = 0 + # Math used requires integers for correct output - as such + # we ensure our inputs are correct. + assert type(tiles_x) is int, "tiles_x must be an integer" + assert type(tiles_y) is int, "tiles_y must be an integer" + assert type(width) is int, "width must be an integer" + assert type(height) is int, "height must be an integer" + out = {"JobInfo": {}, "PluginInfo": {}} cfg = OrderedDict() - w_space = width / tiles_x - h_space = height / tiles_y + w_space = width // tiles_x + h_space = height // tiles_y cfg["TilesCropped"] = "False" + tile = 0 for tile_x in range(1, tiles_x + 1): for tile_y in reversed(range(1, tiles_y + 1)): tile_prefix = "_tile_{}x{}_{}x{}_".format( @@ -769,10 +776,10 @@ def _format_tiles( tiles_x, tiles_y ) - top = int(height - (tile_y * h_space)) - bottom = int(height - ((tile_y - 1) * h_space) - 1) - left = int((tile_x - 1) * w_space) - right = int((tile_x * w_space) - 1) + top = height - (tile_y * h_space) + bottom = height - ((tile_y - 1) * h_space) - 1 + left = (tile_x - 1) * w_space + right = (tile_x * w_space) - 1 # Job Info new_filename = "{}/{}{}".format( From f54f4cf99c824e927e5cbe24c7a02ec0e4a4fc62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Albert?= Date: Sun, 4 Sep 2022 22:43:25 +0200 Subject: [PATCH 085/122] Kitsu : Modification default value for Ignore_projects --- openpype/modules/kitsu/utils/update_op_with_zou.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/openpype/modules/kitsu/utils/update_op_with_zou.py b/openpype/modules/kitsu/utils/update_op_with_zou.py index 199c59053b..26cd125e15 100644 --- a/openpype/modules/kitsu/utils/update_op_with_zou.py +++ b/openpype/modules/kitsu/utils/update_op_with_zou.py @@ -318,7 +318,7 @@ def write_project_to_op(project: dict, dbcon: AvalonMongoDB) -> UpdateOne: ) -def sync_all_projects(login: str, password: str, ignore_projects=[]): +def sync_all_projects(login: str, password: str, ignore_projects: list = None): """Update all OP projects in DB with Zou data. Args: @@ -340,8 +340,9 @@ def sync_all_projects(login: str, password: str, ignore_projects=[]): dbcon.install() all_projects = gazu.project.all_open_projects() for project in all_projects: - if project["name"] not in ignore_projects: - sync_project_from_kitsu(dbcon, project) + if ignore_projects and project["name"] in ignore_projects: + continue + sync_project_from_kitsu(dbcon, project) def sync_project_from_kitsu(dbcon: AvalonMongoDB, project: dict): From fb48faf386c8d53c737ac9a6a00287730c934217 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Albert?= Date: Sun, 4 Sep 2022 22:49:07 +0200 Subject: [PATCH 086/122] Kitsu : Ignore_projects - minor fix - indent --- openpype/modules/kitsu/utils/update_op_with_zou.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/kitsu/utils/update_op_with_zou.py b/openpype/modules/kitsu/utils/update_op_with_zou.py index 26cd125e15..55a7bdc51d 100644 --- a/openpype/modules/kitsu/utils/update_op_with_zou.py +++ b/openpype/modules/kitsu/utils/update_op_with_zou.py @@ -341,7 +341,7 @@ def sync_all_projects(login: str, password: str, ignore_projects: list = None): all_projects = gazu.project.all_open_projects() for project in all_projects: if ignore_projects and project["name"] in ignore_projects: - continue + continue sync_project_from_kitsu(dbcon, project) From d9a150022e1659aec584fe962f9c47e66bfb178d Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Mon, 5 Sep 2022 19:28:48 +0800 Subject: [PATCH 087/122] adding and loading maya mel workspace through openpype project setting --- openpype/hosts/maya/api/pipeline.py | 4 ++-- openpype/hosts/maya/lib.py | 18 ++++++++++++++++++ .../defaults/project_settings/maya.json | 14 ++++++++++++++ .../projects_schema/schema_project_maya.json | 15 +++++++++++++++ 4 files changed, 49 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/maya/api/pipeline.py b/openpype/hosts/maya/api/pipeline.py index f565f6a308..5bf8b67fc2 100644 --- a/openpype/hosts/maya/api/pipeline.py +++ b/openpype/hosts/maya/api/pipeline.py @@ -28,7 +28,7 @@ from openpype.pipeline import ( AVALON_CONTAINER_ID, ) from openpype.pipeline.load import any_outdated_containers -from openpype.hosts.maya.lib import copy_workspace_mel +from openpype.hosts.maya.lib import copy_workspace_mel,load_workspace_mel from . import menu, lib from .workio import ( open_file, @@ -550,7 +550,7 @@ def on_task_changed(): def before_workfile_save(event): workdir_path = event["workdir_path"] if workdir_path: - copy_workspace_mel(workdir_path) + load_workspace_mel(workdir_path) class MayaDirmap(HostDirmap): diff --git a/openpype/hosts/maya/lib.py b/openpype/hosts/maya/lib.py index 6c142053e6..d24f267bbd 100644 --- a/openpype/hosts/maya/lib.py +++ b/openpype/hosts/maya/lib.py @@ -1,6 +1,8 @@ import os import shutil +import json +from openpype.settings import get_current_project_settings def copy_workspace_mel(workdir): # Check that source mel exists @@ -24,3 +26,19 @@ def copy_workspace_mel(workdir): src_filepath, dst_filepath )) shutil.copy(src_filepath, dst_filepath) + + +def load_workspace_mel(workdir): + dst_filepath = os.path.join(workdir, "workspace.mel") + if os.path.exists(dst_filepath): + return + + if not os.path.exists(workdir): + os.makedirs(workdir) + + with open(dst_filepath, "w") as mel_file: + setting = get_current_project_settings() + mel_script = setting["maya"]["mel-workspace"]["scripts"] + for mel in mel_script: + mel_file.write(mel) + mel_file.write("\n") diff --git a/openpype/settings/defaults/project_settings/maya.json b/openpype/settings/defaults/project_settings/maya.json index ac0f161cf2..0a46632042 100644 --- a/openpype/settings/defaults/project_settings/maya.json +++ b/openpype/settings/defaults/project_settings/maya.json @@ -15,6 +15,20 @@ "destination-path": [] } }, + "mel-workspace":{ + "scripts":[ + "workspace -fr \"shaders\" \"renderData/shaders\";", + "workspace -fr \"images\" \"renders\";", + "workspace -fr \"particles\" \"particles\";", + "workspace -fr \"mayaAscii\" \"\";", + "workspace -fr \"mayaBinary\" \"\";", + "workspace -fr \"scene\" \"\";", + "workspace -fr \"alembicCache\" \"cache/alembic\";", + "workspace -fr \"renderData\" \"renderData\";", + "workspace -fr \"sourceImages\" \"sourceimages\";", + "workspace -fr \"fileCache\" \"cache/nCache\";" + ] + }, "scriptsmenu": { "name": "OpenPype Tools", "definition": [ diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_maya.json b/openpype/settings/entities/schemas/projects_schema/schema_project_maya.json index cb380194a7..a774d604ca 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_maya.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_maya.json @@ -53,6 +53,21 @@ } ] }, + { + "type": "dict", + "collapsible": true, + "key": "mel-workspace", + "label": "Maya MEL Workspace", + "is_group": true, + "children": [ + { + "type": "list", + "object_type": "text", + "key": "scripts", + "label": "scripts" + } + ] + }, { "type": "schema", "name": "schema_scriptsmenu" From b88def9aea1fc1a682209ea78edcf5ae87a652e3 Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Mon, 5 Sep 2022 19:40:30 +0800 Subject: [PATCH 088/122] adding and loading maya mel workspace through openpype project setting --- openpype/hosts/maya/lib.py | 27 +------------------ .../defaults/project_settings/maya.json | 2 +- .../projects_schema/schema_project_maya.json | 4 +-- 3 files changed, 4 insertions(+), 29 deletions(-) diff --git a/openpype/hosts/maya/lib.py b/openpype/hosts/maya/lib.py index d24f267bbd..bf06c9ad7d 100644 --- a/openpype/hosts/maya/lib.py +++ b/openpype/hosts/maya/lib.py @@ -1,33 +1,8 @@ import os import shutil -import json from openpype.settings import get_current_project_settings -def copy_workspace_mel(workdir): - # Check that source mel exists - current_dir = os.path.dirname(os.path.abspath(__file__)) - src_filepath = os.path.join(current_dir, "resources", "workspace.mel") - if not os.path.exists(src_filepath): - print("Source mel file does not exist. {}".format(src_filepath)) - return - - # Skip if workspace.mel already exists - dst_filepath = os.path.join(workdir, "workspace.mel") - if os.path.exists(dst_filepath): - return - - # Create workdir if does not exists yet - if not os.path.exists(workdir): - os.makedirs(workdir) - - # Copy file - print("Copying workspace mel \"{}\" -> \"{}\"".format( - src_filepath, dst_filepath - )) - shutil.copy(src_filepath, dst_filepath) - - def load_workspace_mel(workdir): dst_filepath = os.path.join(workdir, "workspace.mel") if os.path.exists(dst_filepath): @@ -38,7 +13,7 @@ def load_workspace_mel(workdir): with open(dst_filepath, "w") as mel_file: setting = get_current_project_settings() - mel_script = setting["maya"]["mel-workspace"]["scripts"] + mel_script = setting["maya"]["mel-workspace"]["definition"] for mel in mel_script: mel_file.write(mel) mel_file.write("\n") diff --git a/openpype/settings/defaults/project_settings/maya.json b/openpype/settings/defaults/project_settings/maya.json index 0a46632042..162732280f 100644 --- a/openpype/settings/defaults/project_settings/maya.json +++ b/openpype/settings/defaults/project_settings/maya.json @@ -16,7 +16,7 @@ } }, "mel-workspace":{ - "scripts":[ + "definition":[ "workspace -fr \"shaders\" \"renderData/shaders\";", "workspace -fr \"images\" \"renders\";", "workspace -fr \"particles\" \"particles\";", diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_maya.json b/openpype/settings/entities/schemas/projects_schema/schema_project_maya.json index a774d604ca..7204ec586a 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_maya.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_maya.json @@ -63,8 +63,8 @@ { "type": "list", "object_type": "text", - "key": "scripts", - "label": "scripts" + "key": "definition", + "label": "definition" } ] }, From 109abb58987b22f6d390d424a27e209eff6b5638 Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Mon, 5 Sep 2022 19:41:09 +0800 Subject: [PATCH 089/122] adding and loading maya mel workspace through openpype project setting --- openpype/hosts/maya/api/pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/hosts/maya/api/pipeline.py b/openpype/hosts/maya/api/pipeline.py index 5bf8b67fc2..4768a9ee4f 100644 --- a/openpype/hosts/maya/api/pipeline.py +++ b/openpype/hosts/maya/api/pipeline.py @@ -28,7 +28,7 @@ from openpype.pipeline import ( AVALON_CONTAINER_ID, ) from openpype.pipeline.load import any_outdated_containers -from openpype.hosts.maya.lib import copy_workspace_mel,load_workspace_mel +from openpype.hosts.maya.lib import load_workspace_mel from . import menu, lib from .workio import ( open_file, From 69d2cf20f5b4889ce674487d2da8fd2a230a093a Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Mon, 5 Sep 2022 19:42:10 +0800 Subject: [PATCH 090/122] adding and loading maya mel workspace through openpype project setting --- openpype/hosts/maya/lib.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openpype/hosts/maya/lib.py b/openpype/hosts/maya/lib.py index bf06c9ad7d..2853789656 100644 --- a/openpype/hosts/maya/lib.py +++ b/openpype/hosts/maya/lib.py @@ -3,6 +3,7 @@ import shutil from openpype.settings import get_current_project_settings + def load_workspace_mel(workdir): dst_filepath = os.path.join(workdir, "workspace.mel") if os.path.exists(dst_filepath): From 5908f49b685a981d08d7ea1ff841d567018a7e76 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 6 Sep 2022 10:56:01 +0200 Subject: [PATCH 091/122] updating README file --- README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b8c04f8b49..a2f442b640 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ It can be built and ran on all common platforms. We develop and test on the foll - **Linux** - **Ubuntu** 20.04 LTS - **Centos** 7 -- **Mac OSX** +- **Mac OSX** - **10.15** Catalina - **11.1** Big Sur (using Rosetta2) @@ -287,6 +287,14 @@ To run tests, execute `.\tools\run_tests(.ps1|.sh)`. **Note that it needs existing virtual environment.** + +Developer tools +------------- + +In case you wish to add your own tools to `.\tools` folder without git tracking, it is possible by adding it with `dev_*` suffix (`dev_clear_pyc(.ps1|.sh)`). + + + ## Contributors ✨ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)): From a0e241b02fbfdee18b8ded65af89eedb343793d2 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 6 Sep 2022 10:58:22 +0200 Subject: [PATCH 092/122] README fix --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a2f442b640..a3d3cf1dbb 100644 --- a/README.md +++ b/README.md @@ -291,7 +291,7 @@ To run tests, execute `.\tools\run_tests(.ps1|.sh)`. Developer tools ------------- -In case you wish to add your own tools to `.\tools` folder without git tracking, it is possible by adding it with `dev_*` suffix (`dev_clear_pyc(.ps1|.sh)`). +In case you wish to add your own tools to `.\tools` folder without git tracking, it is possible by adding it with `dev_*` suffix (example: `dev_clear_pyc(.ps1|.sh)`). From 70a17a8876541de84d98bd8f502421827fd78751 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 6 Sep 2022 11:39:58 +0200 Subject: [PATCH 093/122] hiero: instances detection - timeline no need to retime --- openpype/hosts/hiero/plugins/publish/precollect_instances.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/openpype/hosts/hiero/plugins/publish/precollect_instances.py b/openpype/hosts/hiero/plugins/publish/precollect_instances.py index 0c7dbc1f22..84f2927fc7 100644 --- a/openpype/hosts/hiero/plugins/publish/precollect_instances.py +++ b/openpype/hosts/hiero/plugins/publish/precollect_instances.py @@ -318,10 +318,9 @@ class PrecollectInstances(pyblish.api.ContextPlugin): @staticmethod def create_otio_time_range_from_timeline_item_data(track_item): - speed = track_item.playbackSpeed() timeline = phiero.get_current_sequence() frame_start = int(track_item.timelineIn()) - frame_duration = int((track_item.duration() - 1) / speed) + frame_duration = int(track_item.duration()) fps = timeline.framerate().toFloat() return hiero_export.create_otio_time_range( From 26fbdac8da117c83a71b75ce6315be4044d23942 Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Wed, 7 Sep 2022 16:14:30 +0800 Subject: [PATCH 094/122] adding and loading maya mel workspace through openpype project setting --- openpype/hosts/maya/api/pipeline.py | 5 +++-- openpype/hosts/maya/hooks/pre_copy_mel.py | 5 +++-- openpype/hosts/maya/lib.py | 19 ++++++++++--------- .../defaults/project_anatomy/attributes.json | 3 +-- .../defaults/project_settings/maya.json | 15 +-------------- .../projects_schema/schema_project_maya.json | 17 ++++------------- 6 files changed, 22 insertions(+), 42 deletions(-) diff --git a/openpype/hosts/maya/api/pipeline.py b/openpype/hosts/maya/api/pipeline.py index 4768a9ee4f..4578d6fb39 100644 --- a/openpype/hosts/maya/api/pipeline.py +++ b/openpype/hosts/maya/api/pipeline.py @@ -28,7 +28,7 @@ from openpype.pipeline import ( AVALON_CONTAINER_ID, ) from openpype.pipeline.load import any_outdated_containers -from openpype.hosts.maya.lib import load_workspace_mel +from openpype.hosts.maya.lib import create_workspace_mel from . import menu, lib from .workio import ( open_file, @@ -548,9 +548,10 @@ def on_task_changed(): def before_workfile_save(event): + project_name = os.getenv("AVALON_PROJECT") workdir_path = event["workdir_path"] if workdir_path: - load_workspace_mel(workdir_path) + create_workspace_mel(workdir_path, project_name) class MayaDirmap(HostDirmap): diff --git a/openpype/hosts/maya/hooks/pre_copy_mel.py b/openpype/hosts/maya/hooks/pre_copy_mel.py index b11e18241e..6f90af4b7c 100644 --- a/openpype/hosts/maya/hooks/pre_copy_mel.py +++ b/openpype/hosts/maya/hooks/pre_copy_mel.py @@ -1,5 +1,5 @@ from openpype.lib import PreLaunchHook -from openpype.hosts.maya.lib import copy_workspace_mel +from openpype.hosts.maya.lib import create_workspace_mel class PreCopyMel(PreLaunchHook): @@ -10,9 +10,10 @@ class PreCopyMel(PreLaunchHook): app_groups = ["maya"] def execute(self): + project_name = self.launch_context.env.get("AVALON_PROJECT") workdir = self.launch_context.env.get("AVALON_WORKDIR") if not workdir: self.log.warning("BUG: Workdir is not filled.") return - copy_workspace_mel(workdir) + create_workspace_mel(workdir, project_name) diff --git a/openpype/hosts/maya/lib.py b/openpype/hosts/maya/lib.py index 2853789656..443bf7d10e 100644 --- a/openpype/hosts/maya/lib.py +++ b/openpype/hosts/maya/lib.py @@ -1,10 +1,8 @@ import os -import shutil - -from openpype.settings import get_current_project_settings +from openpype.settings import get_project_settings -def load_workspace_mel(workdir): +def create_workspace_mel(workdir, project_name): dst_filepath = os.path.join(workdir, "workspace.mel") if os.path.exists(dst_filepath): return @@ -12,9 +10,12 @@ def load_workspace_mel(workdir): if not os.path.exists(workdir): os.makedirs(workdir) + project_setting = get_project_settings(project_name) + mel_script = project_setting["maya"].get("mel_workspace") + + # Skip if mel script in settings is empty + if not mel_script: + return + with open(dst_filepath, "w") as mel_file: - setting = get_current_project_settings() - mel_script = setting["maya"]["mel-workspace"]["definition"] - for mel in mel_script: - mel_file.write(mel) - mel_file.write("\n") + mel_file.write(mel_script) diff --git a/openpype/settings/defaults/project_anatomy/attributes.json b/openpype/settings/defaults/project_anatomy/attributes.json index 983ac603f9..bf8bbef8de 100644 --- a/openpype/settings/defaults/project_anatomy/attributes.json +++ b/openpype/settings/defaults/project_anatomy/attributes.json @@ -19,8 +19,7 @@ "blender/2-91", "harmony/20", "photoshop/2021", - "aftereffects/2021", - "unreal/4-26" + "aftereffects/2021" ], "tools_env": [], "active": true diff --git a/openpype/settings/defaults/project_settings/maya.json b/openpype/settings/defaults/project_settings/maya.json index 162732280f..ada69c3730 100644 --- a/openpype/settings/defaults/project_settings/maya.json +++ b/openpype/settings/defaults/project_settings/maya.json @@ -15,20 +15,7 @@ "destination-path": [] } }, - "mel-workspace":{ - "definition":[ - "workspace -fr \"shaders\" \"renderData/shaders\";", - "workspace -fr \"images\" \"renders\";", - "workspace -fr \"particles\" \"particles\";", - "workspace -fr \"mayaAscii\" \"\";", - "workspace -fr \"mayaBinary\" \"\";", - "workspace -fr \"scene\" \"\";", - "workspace -fr \"alembicCache\" \"cache/alembic\";", - "workspace -fr \"renderData\" \"renderData\";", - "workspace -fr \"sourceImages\" \"sourceimages\";", - "workspace -fr \"fileCache\" \"cache/nCache\";" - ] - }, + "mel_workspace": "workspace -fr \"shaders\" \"renderData/shaders\";\nworkspace -fr \"images\" \"renders\";\nworkspace -fr \"particles\" \"particles\";\nworkspace -fr \"mayaAscii\" \"\";\nworkspace -fr \"mayaBinary\" \"\";\nworkspace -fr \"scene\" \"\";\nworkspace -fr \"alembicCache\" \"cache/alembic\";\nworkspace -fr \"renderData\" \"renderData\";\nworkspace -fr \"sourceImages\" \"sourceimages\";\nworkspace -fr \"fileCache\" \"cache/nCache\";\n", "scriptsmenu": { "name": "OpenPype Tools", "definition": [ diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_maya.json b/openpype/settings/entities/schemas/projects_schema/schema_project_maya.json index 7204ec586a..978de56a51 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_maya.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_maya.json @@ -54,19 +54,10 @@ ] }, { - "type": "dict", - "collapsible": true, - "key": "mel-workspace", - "label": "Maya MEL Workspace", - "is_group": true, - "children": [ - { - "type": "list", - "object_type": "text", - "key": "definition", - "label": "definition" - } - ] + "type": "text", + "multiline" : true, + "key": "mel_workspace", + "label": "Maya MEL Workspace" }, { "type": "schema", From a9b69536cac401221cacaaa3155c4f9a7be682b8 Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Wed, 7 Sep 2022 16:44:29 +0800 Subject: [PATCH 095/122] adding and loading maya mel workspace through openpype project setting --- openpype/settings/defaults/project_anatomy/attributes.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openpype/settings/defaults/project_anatomy/attributes.json b/openpype/settings/defaults/project_anatomy/attributes.json index bf8bbef8de..983ac603f9 100644 --- a/openpype/settings/defaults/project_anatomy/attributes.json +++ b/openpype/settings/defaults/project_anatomy/attributes.json @@ -19,7 +19,8 @@ "blender/2-91", "harmony/20", "photoshop/2021", - "aftereffects/2021" + "aftereffects/2021", + "unreal/4-26" ], "tools_env": [], "active": true From decc11251854f60db02531f93e2b8fbd4d3fa7ec Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Wed, 7 Sep 2022 17:06:47 +0800 Subject: [PATCH 096/122] load and edit mel workspace within the Openpype project settings --- .../settings/defaults/project_settings/maya.json | 2 +- .../schemas/projects_schema/schema_project_maya.json | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/openpype/settings/defaults/project_settings/maya.json b/openpype/settings/defaults/project_settings/maya.json index ada69c3730..bb96fcf741 100644 --- a/openpype/settings/defaults/project_settings/maya.json +++ b/openpype/settings/defaults/project_settings/maya.json @@ -1,4 +1,5 @@ { + "mel_workspace": "workspace -fr \"shaders\" \"renderData/shaders\";\nworkspace -fr \"images\" \"renders\";\nworkspace -fr \"particles\" \"particles\";\nworkspace -fr \"mayaAscii\" \"\";\nworkspace -fr \"mayaBinary\" \"\";\nworkspace -fr \"scene\" \"\";\nworkspace -fr \"alembicCache\" \"cache/alembic\";\nworkspace -fr \"renderData\" \"renderData\";\nworkspace -fr \"sourceImages\" \"sourceimages\";\nworkspace -fr \"fileCache\" \"cache/nCache\";\n", "ext_mapping": { "model": "ma", "mayaAscii": "ma", @@ -15,7 +16,6 @@ "destination-path": [] } }, - "mel_workspace": "workspace -fr \"shaders\" \"renderData/shaders\";\nworkspace -fr \"images\" \"renders\";\nworkspace -fr \"particles\" \"particles\";\nworkspace -fr \"mayaAscii\" \"\";\nworkspace -fr \"mayaBinary\" \"\";\nworkspace -fr \"scene\" \"\";\nworkspace -fr \"alembicCache\" \"cache/alembic\";\nworkspace -fr \"renderData\" \"renderData\";\nworkspace -fr \"sourceImages\" \"sourceimages\";\nworkspace -fr \"fileCache\" \"cache/nCache\";\n", "scriptsmenu": { "name": "OpenPype Tools", "definition": [ diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_maya.json b/openpype/settings/entities/schemas/projects_schema/schema_project_maya.json index 978de56a51..a54f8e6e4f 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_maya.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_maya.json @@ -5,6 +5,12 @@ "label": "Maya", "is_file": true, "children": [ + { + "type": "text", + "multiline" : true, + "key": "mel_workspace", + "label": "Maya MEL Workspace" + }, { "type": "dict-modifiable", "key": "ext_mapping", @@ -53,12 +59,6 @@ } ] }, - { - "type": "text", - "multiline" : true, - "key": "mel_workspace", - "label": "Maya MEL Workspace" - }, { "type": "schema", "name": "schema_scriptsmenu" From e1c17c71d30a38bcfda292393e425651c4c5a6d4 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 7 Sep 2022 12:11:06 +0200 Subject: [PATCH 097/122] fix variable name --- openpype/pipeline/template_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/pipeline/template_data.py b/openpype/pipeline/template_data.py index bab46a627d..627eba5c3d 100644 --- a/openpype/pipeline/template_data.py +++ b/openpype/pipeline/template_data.py @@ -53,7 +53,7 @@ def get_project_template_data(project_doc=None, project_name=None): project_name = project_doc["name"] if not project_doc: - project_code = get_project(project_name, fields=["data.code"]) + project_doc = get_project(project_name, fields=["data.code"]) project_code = project_doc.get("data", {}).get("code") return { From 509c209093d6fd971ec3631e662a8e34a31c5717 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 7 Sep 2022 12:40:34 +0200 Subject: [PATCH 098/122] fix status handling --- igniter/install_dialog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/igniter/install_dialog.py b/igniter/install_dialog.py index b09529f5c5..c7e9ef74c5 100644 --- a/igniter/install_dialog.py +++ b/igniter/install_dialog.py @@ -389,7 +389,7 @@ class InstallDialog(QtWidgets.QDialog): def _installation_finished(self): status = self._install_thread.result() - if status >= 0: + if status is not None and status >= 0: self._update_progress(100) QtWidgets.QApplication.processEvents() self.done(3) From d6ab41887a37e1bfb4e7f9a1f430daef93182faa Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 7 Sep 2022 13:41:30 +0200 Subject: [PATCH 099/122] added comment to code --- igniter/install_dialog.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/igniter/install_dialog.py b/igniter/install_dialog.py index c7e9ef74c5..65ddd58735 100644 --- a/igniter/install_dialog.py +++ b/igniter/install_dialog.py @@ -388,6 +388,9 @@ class InstallDialog(QtWidgets.QDialog): install_thread.start() def _installation_finished(self): + # TODO we should find out why status can be set to 'None'? + # - 'InstallThread.run' should handle all cases so not sure where + # that come from status = self._install_thread.result() if status is not None and status >= 0: self._update_progress(100) From 57aa1e6659ab9552c1980dbebcd8b64535469f39 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 7 Sep 2022 14:49:55 +0200 Subject: [PATCH 100/122] implementing not retimed working frame range and retimed handles switch --- openpype/hosts/flame/api/plugin.py | 10 ++++++ .../flame/plugins/create/create_shot_clip.py | 16 ++++++++++ .../publish/collect_timeline_instances.py | 4 +++ .../publish/extract_subset_resources.py | 32 +++++++++++++------ .../publish/collect_otio_frame_ranges.py | 6 ++++ 5 files changed, 58 insertions(+), 10 deletions(-) diff --git a/openpype/hosts/flame/api/plugin.py b/openpype/hosts/flame/api/plugin.py index efbabb6a55..145b1f0921 100644 --- a/openpype/hosts/flame/api/plugin.py +++ b/openpype/hosts/flame/api/plugin.py @@ -361,6 +361,8 @@ class PublishableClip: index_from_segment_default = False use_shot_name_default = False include_handles_default = False + retimed_handles_default = True + retimed_framerange_default = True def __init__(self, segment, **kwargs): self.rename_index = kwargs["rename_index"] @@ -496,6 +498,14 @@ class PublishableClip: "audio", {}).get("value") or False self.include_handles = self.ui_inputs.get( "includeHandles", {}).get("value") or self.include_handles_default + self.retimed_handles = ( + self.ui_inputs.get("retimedHandles", {}).get("value") + or self.retimed_handles_default + ) + self.retimed_framerange = ( + self.ui_inputs.get("retimedFramerange", {}).get("value") + or self.retimed_framerange_default + ) # build subset name from layer name if self.subset_name == "[ track name ]": diff --git a/openpype/hosts/flame/plugins/create/create_shot_clip.py b/openpype/hosts/flame/plugins/create/create_shot_clip.py index fa239ea420..b03a39a7ca 100644 --- a/openpype/hosts/flame/plugins/create/create_shot_clip.py +++ b/openpype/hosts/flame/plugins/create/create_shot_clip.py @@ -276,6 +276,22 @@ class CreateShotClip(opfapi.Creator): "target": "tag", "toolTip": "By default handles are excluded", # noqa "order": 3 + }, + "retimedHandles": { + "value": True, + "type": "QCheckBox", + "label": "Retimed handles", + "target": "tag", + "toolTip": "By default handles are retimed.", # noqa + "order": 4 + }, + "retimedFramerange": { + "value": True, + "type": "QCheckBox", + "label": "Retimed framerange", + "target": "tag", + "toolTip": "By default framerange is retimed.", # noqa + "order": 5 } } } diff --git a/openpype/hosts/flame/plugins/publish/collect_timeline_instances.py b/openpype/hosts/flame/plugins/publish/collect_timeline_instances.py index 992db62c75..d6ff13b059 100644 --- a/openpype/hosts/flame/plugins/publish/collect_timeline_instances.py +++ b/openpype/hosts/flame/plugins/publish/collect_timeline_instances.py @@ -131,6 +131,10 @@ class CollectTimelineInstances(pyblish.api.ContextPlugin): "fps": self.fps, "workfileFrameStart": workfile_start, "sourceFirstFrame": int(first_frame), + "notRetimedHandles": ( + not marker_data.get("retimedHandles")), + "notRetimedFramerange": ( + not marker_data.get("retimedFramerange")), "path": file_path, "flameAddTasks": self.add_tasks, "tasks": { diff --git a/openpype/hosts/flame/plugins/publish/extract_subset_resources.py b/openpype/hosts/flame/plugins/publish/extract_subset_resources.py index 3e1e8db986..1af6b00654 100644 --- a/openpype/hosts/flame/plugins/publish/extract_subset_resources.py +++ b/openpype/hosts/flame/plugins/publish/extract_subset_resources.py @@ -90,26 +90,38 @@ class ExtractSubsetResources(openpype.api.Extractor): handle_end = instance.data["handleEnd"] handles = max(handle_start, handle_end) include_handles = instance.data.get("includeHandles") + retimed_handles = instance.data.get("retimedHandles") # get media source range with handles source_start_handles = instance.data["sourceStartH"] source_end_handles = instance.data["sourceEndH"] - # retime if needed + + # retime if needed if r_speed != 1.0: - source_start_handles = ( - instance.data["sourceStart"] - r_handle_start) - source_end_handles = ( - source_start_handles - + (r_source_dur - 1) - + r_handle_start - + r_handle_end - ) + if retimed_handles: + # handles are retimed + source_start_handles = ( + instance.data["sourceStart"] - r_handle_start) + source_end_handles = ( + source_start_handles + + (r_source_dur - 1) + + r_handle_start + + r_handle_end + ) + else: + # handles are not retimed + source_end_handles = ( + source_start_handles + + (r_source_dur - 1) + + handle_start + + handle_end + ) # get frame range with handles for representation range frame_start_handle = frame_start - handle_start repre_frame_start = frame_start_handle if include_handles: - if r_speed == 1.0: + if r_speed == 1.0 or not retimed_handles: frame_start_handle = frame_start else: frame_start_handle = ( diff --git a/openpype/plugins/publish/collect_otio_frame_ranges.py b/openpype/plugins/publish/collect_otio_frame_ranges.py index 40e89e29bc..40a3fa6978 100644 --- a/openpype/plugins/publish/collect_otio_frame_ranges.py +++ b/openpype/plugins/publish/collect_otio_frame_ranges.py @@ -29,6 +29,7 @@ class CollectOtioFrameRanges(pyblish.api.InstancePlugin): # get basic variables otio_clip = instance.data["otioClip"] workfile_start = instance.data["workfileFrameStart"] + not_retime_framerange = instance.data.get("notRetimedFramerange") # get ranges otio_tl_range = otio_clip.range_in_parent() @@ -54,6 +55,11 @@ class CollectOtioFrameRanges(pyblish.api.InstancePlugin): frame_end = frame_start + otio.opentime.to_frames( otio_tl_range.duration, otio_tl_range.duration.rate) - 1 + # in case of retimed clip and frame range should not be retimed + if not_retime_framerange: + frame_end = frame_start + otio.opentime.to_frames( + otio_src_range.duration, otio_src_range.duration.rate) - 1 + data = { "frameStart": frame_start, "frameEnd": frame_end, From d4eeabad7e0883bb2f2d31e0b8479e4b547ba7cc Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Wed, 7 Sep 2022 21:17:49 +0800 Subject: [PATCH 101/122] adding and loading maya mel workspace through openpype project setting --- openpype/hosts/maya/api/pipeline.py | 6 +++--- openpype/hosts/maya/lib.py | 5 +++-- .../schemas/projects_schema/schema_project_maya.json | 1 + 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/openpype/hosts/maya/api/pipeline.py b/openpype/hosts/maya/api/pipeline.py index 4578d6fb39..6012d82263 100644 --- a/openpype/hosts/maya/api/pipeline.py +++ b/openpype/hosts/maya/api/pipeline.py @@ -59,7 +59,7 @@ class MayaHost(HostBase, IWorkfileHost, ILoadHost): self._op_events = {} def install(self): - project_settings = get_project_settings(os.getenv("AVALON_PROJECT")) + project_settings = get_project_settings(legacy_io.active_project()) # process path mapping dirmap_processor = MayaDirmap("maya", project_settings) dirmap_processor.process_dirmap() @@ -536,7 +536,7 @@ def on_task_changed(): lib.update_content_on_context_change() msg = " project: {}\n asset: {}\n task:{}".format( - legacy_io.Session["AVALON_PROJECT"], + legacy_io.active_project(), legacy_io.Session["AVALON_ASSET"], legacy_io.Session["AVALON_TASK"] ) @@ -548,7 +548,7 @@ def on_task_changed(): def before_workfile_save(event): - project_name = os.getenv("AVALON_PROJECT") + project_name = legacy_io.active_project() workdir_path = event["workdir_path"] if workdir_path: create_workspace_mel(workdir_path, project_name) diff --git a/openpype/hosts/maya/lib.py b/openpype/hosts/maya/lib.py index 443bf7d10e..e466850810 100644 --- a/openpype/hosts/maya/lib.py +++ b/openpype/hosts/maya/lib.py @@ -1,6 +1,6 @@ import os from openpype.settings import get_project_settings - +from openpype.api import Logger def create_workspace_mel(workdir, project_name): dst_filepath = os.path.join(workdir, "workspace.mel") @@ -15,7 +15,8 @@ def create_workspace_mel(workdir, project_name): # Skip if mel script in settings is empty if not mel_script: - return + log = Logger.get_logger("create_workspace_mel") + log.debug("File 'workspace.mel' not created. Settings value is empty.") with open(dst_filepath, "w") as mel_file: mel_file.write(mel_script) diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_maya.json b/openpype/settings/entities/schemas/projects_schema/schema_project_maya.json index a54f8e6e4f..72c974642f 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_maya.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_maya.json @@ -8,6 +8,7 @@ { "type": "text", "multiline" : true, + "use_label_wrap": true, "key": "mel_workspace", "label": "Maya MEL Workspace" }, From a6d7df1423fdd5ec37744a34042e01a76776f263 Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Wed, 7 Sep 2022 21:19:15 +0800 Subject: [PATCH 102/122] adding and loading maya mel workspace through openpype project setting --- openpype/hosts/maya/lib.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openpype/hosts/maya/lib.py b/openpype/hosts/maya/lib.py index e466850810..e07e174dd6 100644 --- a/openpype/hosts/maya/lib.py +++ b/openpype/hosts/maya/lib.py @@ -2,6 +2,7 @@ import os from openpype.settings import get_project_settings from openpype.api import Logger + def create_workspace_mel(workdir, project_name): dst_filepath = os.path.join(workdir, "workspace.mel") if os.path.exists(dst_filepath): From b7256e7c19ba376e438a88ca5d0b4a9609a44423 Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Wed, 7 Sep 2022 21:49:02 +0800 Subject: [PATCH 103/122] adding and loading maya mel workspace through openpype project setting --- openpype/hosts/maya/lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/hosts/maya/lib.py b/openpype/hosts/maya/lib.py index e07e174dd6..6f7bb8f986 100644 --- a/openpype/hosts/maya/lib.py +++ b/openpype/hosts/maya/lib.py @@ -1,6 +1,6 @@ import os from openpype.settings import get_project_settings -from openpype.api import Logger +from openpype.lib import Logger def create_workspace_mel(workdir, project_name): From d2b3c80fb93acf26f9d61a6164dadf2941cd7930 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 7 Sep 2022 16:02:38 +0200 Subject: [PATCH 104/122] improving variable name --- openpype/plugins/publish/collect_otio_frame_ranges.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/plugins/publish/collect_otio_frame_ranges.py b/openpype/plugins/publish/collect_otio_frame_ranges.py index 40a3fa6978..cfb0318950 100644 --- a/openpype/plugins/publish/collect_otio_frame_ranges.py +++ b/openpype/plugins/publish/collect_otio_frame_ranges.py @@ -29,7 +29,7 @@ class CollectOtioFrameRanges(pyblish.api.InstancePlugin): # get basic variables otio_clip = instance.data["otioClip"] workfile_start = instance.data["workfileFrameStart"] - not_retime_framerange = instance.data.get("notRetimedFramerange") + workfile_source_duration = instance.data.get("notRetimedFramerange") # get ranges otio_tl_range = otio_clip.range_in_parent() @@ -56,7 +56,7 @@ class CollectOtioFrameRanges(pyblish.api.InstancePlugin): otio_tl_range.duration, otio_tl_range.duration.rate) - 1 # in case of retimed clip and frame range should not be retimed - if not_retime_framerange: + if workfile_source_duration: frame_end = frame_start + otio.opentime.to_frames( otio_src_range.duration, otio_src_range.duration.rate) - 1 From 9bcd86bac7ca0294fc41bc3d2465166b3b5e8861 Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Wed, 7 Sep 2022 22:12:16 +0800 Subject: [PATCH 105/122] load and edit mel workspace within the Openpype project settings --- openpype/hosts/maya/lib.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openpype/hosts/maya/lib.py b/openpype/hosts/maya/lib.py index 6f7bb8f986..ffb2f0b27c 100644 --- a/openpype/hosts/maya/lib.py +++ b/openpype/hosts/maya/lib.py @@ -18,6 +18,7 @@ def create_workspace_mel(workdir, project_name): if not mel_script: log = Logger.get_logger("create_workspace_mel") log.debug("File 'workspace.mel' not created. Settings value is empty.") + return with open(dst_filepath, "w") as mel_file: mel_file.write(mel_script) From 5627c8ec627e4157e45eaeeb5a58e12216979f7c Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 8 Sep 2022 11:23:56 +0200 Subject: [PATCH 106/122] enabled pixmap scaling in tray --- openpype/tools/tray/pype_tray.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/openpype/tools/tray/pype_tray.py b/openpype/tools/tray/pype_tray.py index 85bc00ead6..c32a074fd1 100644 --- a/openpype/tools/tray/pype_tray.py +++ b/openpype/tools/tray/pype_tray.py @@ -778,6 +778,14 @@ def main(): if not app: app = QtWidgets.QApplication([]) + for attr_name in ( + "AA_EnableHighDpiScaling", + "AA_UseHighDpiPixmaps" + ): + attr = getattr(QtCore.Qt, attr_name, None) + if attr is not None: + app.setAttribute(attr) + starter = PypeTrayStarter(app) # TODO remove when pype.exe will have an icon From 4f3accee1ac84991543493a32768c1aa99672035 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 8 Sep 2022 11:34:33 +0200 Subject: [PATCH 107/122] change "hierarchy" key to "nodesHierarchy" in maya --- openpype/hosts/maya/plugins/publish/collect_assembly.py | 2 +- openpype/hosts/maya/plugins/publish/extract_assembly.py | 2 +- .../hosts/maya/plugins/publish/validate_assembly_transforms.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/collect_assembly.py b/openpype/hosts/maya/plugins/publish/collect_assembly.py index 1a65bf1fde..2aef9ab908 100644 --- a/openpype/hosts/maya/plugins/publish/collect_assembly.py +++ b/openpype/hosts/maya/plugins/publish/collect_assembly.py @@ -70,7 +70,7 @@ class CollectAssembly(pyblish.api.InstancePlugin): data[representation_id].append(instance_data) instance.data["scenedata"] = dict(data) - instance.data["hierarchy"] = list(set(hierarchy_nodes)) + instance.data["nodesHierarchy"] = list(set(hierarchy_nodes)) def get_file_rule(self, rule): return mel.eval('workspace -query -fileRuleEntry "{}"'.format(rule)) diff --git a/openpype/hosts/maya/plugins/publish/extract_assembly.py b/openpype/hosts/maya/plugins/publish/extract_assembly.py index 482930b76e..120805894e 100644 --- a/openpype/hosts/maya/plugins/publish/extract_assembly.py +++ b/openpype/hosts/maya/plugins/publish/extract_assembly.py @@ -33,7 +33,7 @@ class ExtractAssembly(openpype.api.Extractor): json.dump(instance.data["scenedata"], filepath, ensure_ascii=False) self.log.info("Extracting point cache ..") - cmds.select(instance.data["hierarchy"]) + cmds.select(instance.data["nodesHierarchy"]) # Run basic alembic exporter extract_alembic(file=hierarchy_path, diff --git a/openpype/hosts/maya/plugins/publish/validate_assembly_transforms.py b/openpype/hosts/maya/plugins/publish/validate_assembly_transforms.py index f793846555..fb25b617be 100644 --- a/openpype/hosts/maya/plugins/publish/validate_assembly_transforms.py +++ b/openpype/hosts/maya/plugins/publish/validate_assembly_transforms.py @@ -48,7 +48,7 @@ class ValidateAssemblyModelTransforms(pyblish.api.InstancePlugin): from openpype.hosts.maya.api import lib # Get all transforms in the loaded containers - container_roots = cmds.listRelatives(instance.data["hierarchy"], + container_roots = cmds.listRelatives(instance.data["nodesHierarchy"], children=True, type="transform", fullPath=True) From 01d3b0b0b0e2ed72b4ff509d4e0685f44086ef85 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 8 Sep 2022 13:30:27 +0200 Subject: [PATCH 108/122] add logs about missing attributes --- openpype/tools/tray/pype_tray.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/openpype/tools/tray/pype_tray.py b/openpype/tools/tray/pype_tray.py index c32a074fd1..348573a191 100644 --- a/openpype/tools/tray/pype_tray.py +++ b/openpype/tools/tray/pype_tray.py @@ -9,11 +9,11 @@ import platform from Qt import QtCore, QtGui, QtWidgets import openpype.version -from openpype.api import ( - resources, - get_system_settings +from openpype import resources, style +from openpype.lib import ( + get_openpype_execute_args, + Logger, ) -from openpype.lib import get_openpype_execute_args, Logger from openpype.lib.openpype_version import ( op_version_control_available, get_expected_version, @@ -25,8 +25,8 @@ from openpype.lib.openpype_version import ( get_openpype_version, ) from openpype.modules import TrayModulesManager -from openpype import style from openpype.settings import ( + get_system_settings, SystemSettings, ProjectSettings, DefaultsNotDefined @@ -774,6 +774,7 @@ class PypeTrayStarter(QtCore.QObject): def main(): + log = Logger.get_logger(__name__) app = QtWidgets.QApplication.instance() if not app: app = QtWidgets.QApplication([]) @@ -783,7 +784,12 @@ def main(): "AA_UseHighDpiPixmaps" ): attr = getattr(QtCore.Qt, attr_name, None) - if attr is not None: + if attr is None: + log.debug(( + "Missing QtCore.Qt attribute \"{}\"." + " UI quality may be affected." + ).format(attr_name)) + else: app.setAttribute(attr) starter = PypeTrayStarter(app) From d7a768f718944c9322ae30e9167ea64b655be5e5 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 8 Sep 2022 15:51:24 +0200 Subject: [PATCH 109/122] turn plugin off by default --- openpype/settings/defaults/project_settings/global.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/settings/defaults/project_settings/global.json b/openpype/settings/defaults/project_settings/global.json index 9258343440..99a2e16a7c 100644 --- a/openpype/settings/defaults/project_settings/global.json +++ b/openpype/settings/defaults/project_settings/global.json @@ -4,7 +4,7 @@ "follow_workfile_version": false }, "CollectAudio": { - "enabled": true, + "enabled": false, "audio_subset_name": "audioMain" }, "CollectSceneVersion": { From 49dff63f08207eea0218cf37e4824795d08e3895 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Thu, 8 Sep 2022 17:50:55 +0200 Subject: [PATCH 110/122] Fix detection of workfile instance --- openpype/modules/deadline/abstract_submit_deadline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py index f698b7688e..512ff800ee 100644 --- a/openpype/modules/deadline/abstract_submit_deadline.py +++ b/openpype/modules/deadline/abstract_submit_deadline.py @@ -519,7 +519,7 @@ class AbstractSubmitDeadline(pyblish.api.InstancePlugin): instance = self._instance workfile_instance = self._get_workfile_instance(instance.context) - if not workfile_instance: + if workfile_instance is None: return # determine published path from Anatomy. From 3943d74f3ea3b72d08f2d2e114a30b4a5e00a515 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 9 Sep 2022 11:55:05 +0200 Subject: [PATCH 111/122] flame: adding batch action hook --- openpype/hosts/flame/api/__init__.py | 4 +- openpype/hosts/flame/api/menu.py | 50 +++++++++++++++++++ .../hosts/flame/startup/openpype_in_flame.py | 13 +++++ 3 files changed, 66 insertions(+), 1 deletion(-) diff --git a/openpype/hosts/flame/api/__init__.py b/openpype/hosts/flame/api/__init__.py index 76c1c93379..7da91d41e4 100644 --- a/openpype/hosts/flame/api/__init__.py +++ b/openpype/hosts/flame/api/__init__.py @@ -51,7 +51,8 @@ from .pipeline import ( ) from .menu import ( FlameMenuProjectConnect, - FlameMenuTimeline + FlameMenuTimeline, + FlameMenuBatch ) from .plugin import ( Creator, @@ -131,6 +132,7 @@ __all__ = [ # menu "FlameMenuProjectConnect", "FlameMenuTimeline", + "FlameMenuBatch", # plugin "Creator", diff --git a/openpype/hosts/flame/api/menu.py b/openpype/hosts/flame/api/menu.py index 7f1a6a24e2..a822059930 100644 --- a/openpype/hosts/flame/api/menu.py +++ b/openpype/hosts/flame/api/menu.py @@ -201,3 +201,53 @@ class FlameMenuTimeline(_FlameMenuApp): if self.flame: self.flame.execute_shortcut('Rescan Python Hooks') self.log.info('Rescan Python Hooks') + + +class FlameMenuBatch(_FlameMenuApp): + + # flameMenuProjectconnect app takes care of the preferences dialog as well + + def __init__(self, framework): + _FlameMenuApp.__init__(self, framework) + + def __getattr__(self, name): + def method(*args, **kwargs): + project = self.dynamic_menu_data.get(name) + if project: + self.link_project(project) + return method + + def build_menu(self): + if not self.flame: + return [] + + menu = deepcopy(self.menu) + + menu['actions'].append({ + "name": "Load...", + "execute": lambda x: self.tools_helper.show_loader() + }) + menu['actions'].append({ + "name": "Manage...", + "execute": lambda x: self.tools_helper.show_scene_inventory() + }) + menu['actions'].append({ + "name": "Library...", + "execute": lambda x: self.tools_helper.show_library_loader() + }) + return menu + + def refresh(self, *args, **kwargs): + self.rescan() + + def rescan(self, *args, **kwargs): + if not self.flame: + try: + import flame + self.flame = flame + except ImportError: + self.flame = None + + if self.flame: + self.flame.execute_shortcut('Rescan Python Hooks') + self.log.info('Rescan Python Hooks') diff --git a/openpype/hosts/flame/startup/openpype_in_flame.py b/openpype/hosts/flame/startup/openpype_in_flame.py index f2ac23b19e..60f6612b7f 100644 --- a/openpype/hosts/flame/startup/openpype_in_flame.py +++ b/openpype/hosts/flame/startup/openpype_in_flame.py @@ -73,6 +73,8 @@ def load_apps(): opfapi.FlameMenuProjectConnect(opfapi.CTX.app_framework)) opfapi.CTX.flame_apps.append( opfapi.FlameMenuTimeline(opfapi.CTX.app_framework)) + opfapi.CTX.flame_apps.append( + opfapi.FlameMenuBatch(opfapi.CTX.app_framework)) opfapi.CTX.app_framework.log.info("Apps are loaded") @@ -191,3 +193,14 @@ def get_timeline_custom_ui_actions(): openpype_install() return _build_app_menu("FlameMenuTimeline") + +def get_batch_custom_ui_actions(): + """Hook to create submenu in batch + + Returns: + list: menu object + """ + # install openpype and the host + openpype_install() + + return _build_app_menu("FlameMenuBatch") \ No newline at end of file From c622eb7a59986ebe003205d66cd2ae101e1b23eb Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 9 Sep 2022 13:54:33 +0200 Subject: [PATCH 112/122] flame: add ui to project media panel --- openpype/hosts/flame/api/__init__.py | 4 ++-- openpype/hosts/flame/api/menu.py | 2 +- openpype/hosts/flame/startup/openpype_in_flame.py | 15 ++++++++++++++- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/openpype/hosts/flame/api/__init__.py b/openpype/hosts/flame/api/__init__.py index 7da91d41e4..c00ee958b6 100644 --- a/openpype/hosts/flame/api/__init__.py +++ b/openpype/hosts/flame/api/__init__.py @@ -52,7 +52,7 @@ from .pipeline import ( from .menu import ( FlameMenuProjectConnect, FlameMenuTimeline, - FlameMenuBatch + FlameMenuUniversal ) from .plugin import ( Creator, @@ -132,7 +132,7 @@ __all__ = [ # menu "FlameMenuProjectConnect", "FlameMenuTimeline", - "FlameMenuBatch", + "FlameMenuUniversal", # plugin "Creator", diff --git a/openpype/hosts/flame/api/menu.py b/openpype/hosts/flame/api/menu.py index a822059930..f72a352bba 100644 --- a/openpype/hosts/flame/api/menu.py +++ b/openpype/hosts/flame/api/menu.py @@ -203,7 +203,7 @@ class FlameMenuTimeline(_FlameMenuApp): self.log.info('Rescan Python Hooks') -class FlameMenuBatch(_FlameMenuApp): +class FlameMenuUniversal(_FlameMenuApp): # flameMenuProjectconnect app takes care of the preferences dialog as well diff --git a/openpype/hosts/flame/startup/openpype_in_flame.py b/openpype/hosts/flame/startup/openpype_in_flame.py index 60f6612b7f..9fdc30db5d 100644 --- a/openpype/hosts/flame/startup/openpype_in_flame.py +++ b/openpype/hosts/flame/startup/openpype_in_flame.py @@ -194,6 +194,7 @@ def get_timeline_custom_ui_actions(): return _build_app_menu("FlameMenuTimeline") + def get_batch_custom_ui_actions(): """Hook to create submenu in batch @@ -203,4 +204,16 @@ def get_batch_custom_ui_actions(): # install openpype and the host openpype_install() - return _build_app_menu("FlameMenuBatch") \ No newline at end of file + return _build_app_menu("FlameMenuUniversal") + + +def get_media_panel_custom_ui_actions(): + """Hook to create submenu in desktop + + Returns: + list: menu object + """ + # install openpype and the host + openpype_install() + + return _build_app_menu("FlameMenuUniversal") From 7b78e09eaec1c7e243782c13079ff70ca6c06d23 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 9 Sep 2022 13:59:15 +0200 Subject: [PATCH 113/122] fixing name of class --- openpype/hosts/flame/startup/openpype_in_flame.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/hosts/flame/startup/openpype_in_flame.py b/openpype/hosts/flame/startup/openpype_in_flame.py index 9fdc30db5d..d07aaa6b7d 100644 --- a/openpype/hosts/flame/startup/openpype_in_flame.py +++ b/openpype/hosts/flame/startup/openpype_in_flame.py @@ -74,7 +74,7 @@ def load_apps(): opfapi.CTX.flame_apps.append( opfapi.FlameMenuTimeline(opfapi.CTX.app_framework)) opfapi.CTX.flame_apps.append( - opfapi.FlameMenuBatch(opfapi.CTX.app_framework)) + opfapi.FlameMenuUniversal(opfapi.CTX.app_framework)) opfapi.CTX.app_framework.log.info("Apps are loaded") From 42f575fca94722a25ae462a5112a843799d9aad1 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 9 Sep 2022 14:23:36 +0200 Subject: [PATCH 114/122] actions are expected as list so each application in group is stored if force not open workfile is enabled --- openpype/tools/launcher/models.py | 46 +++++++++++++++++++------------ 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/openpype/tools/launcher/models.py b/openpype/tools/launcher/models.py index 6d40d21f96..6e3b531018 100644 --- a/openpype/tools/launcher/models.py +++ b/openpype/tools/launcher/models.py @@ -281,18 +281,25 @@ class ActionModel(QtGui.QStandardItemModel): if not action_item: return - action = action_item.data(ACTION_ROLE) - actual_data = self._prepare_compare_data(action) + actions = action_item.data(ACTION_ROLE) + if not isinstance(actions, list): + actions = [actions] + + action_actions_data = [ + self._prepare_compare_data(action) + for action in actions + ] stored = self.launcher_registry.get_item("force_not_open_workfile") - if is_checked: - stored.append(actual_data) - else: - final_values = [] - for config in stored: - if config != actual_data: - final_values.append(config) - stored = final_values + for actual_data in action_actions_data: + if is_checked: + stored.append(actual_data) + else: + final_values = [] + for config in stored: + if config != actual_data: + final_values.append(config) + stored = final_values self.launcher_registry.set_item("force_not_open_workfile", stored) self.launcher_registry._get_item.cache_clear() @@ -329,21 +336,24 @@ class ActionModel(QtGui.QStandardItemModel): item (QStandardItem) stored (list) of dict """ - action = item.data(ACTION_ROLE) - if not self.is_application_action(action): + + actions = item.data(ACTION_ROLE) + if not isinstance(actions, list): + actions = [actions] + + if not self.is_application_action(actions[0]): return False - actual_data = self._prepare_compare_data(action) + action_actions_data = [ + self._prepare_compare_data(action) + for action in actions + ] for config in stored: - if config == actual_data: + if config in action_actions_data: return True - return False def _prepare_compare_data(self, action): - if isinstance(action, list) and action: - action = action[0] - compare_data = {} if action and action.label: compare_data = { From 98c065cb8b3c8f75ae0479da1a3287ebbe0b22d6 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 9 Sep 2022 14:24:02 +0200 Subject: [PATCH 115/122] change "start_last_workfile" when triggered from group --- openpype/tools/launcher/widgets.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/openpype/tools/launcher/widgets.py b/openpype/tools/launcher/widgets.py index 62599664fe..774ceb659d 100644 --- a/openpype/tools/launcher/widgets.py +++ b/openpype/tools/launcher/widgets.py @@ -312,11 +312,12 @@ class ActionBar(QtWidgets.QWidget): is_group = index.data(GROUP_ROLE) is_variant_group = index.data(VARIANT_GROUP_ROLE) + force_not_open_workfile = index.data(FORCE_NOT_OPEN_WORKFILE_ROLE) if not is_group and not is_variant_group: action = index.data(ACTION_ROLE) # Change data of application action if issubclass(action, ApplicationAction): - if index.data(FORCE_NOT_OPEN_WORKFILE_ROLE): + if force_not_open_workfile: action.data["start_last_workfile"] = False else: action.data.pop("start_last_workfile", None) @@ -385,10 +386,18 @@ class ActionBar(QtWidgets.QWidget): menu.addMenu(sub_menu) result = menu.exec_(QtGui.QCursor.pos()) - if result: - action = actions_mapping[result] - self._start_animation(index) - self.action_clicked.emit(action) + if not result: + return + + action = actions_mapping[result] + if issubclass(action, ApplicationAction): + if force_not_open_workfile: + action.data["start_last_workfile"] = False + else: + action.data.pop("start_last_workfile", None) + + self._start_animation(index) + self.action_clicked.emit(action) class ActionHistory(QtWidgets.QPushButton): From 113210a781ed5affd6e92f72d075a9522a07c0f0 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 9 Sep 2022 17:21:53 +0200 Subject: [PATCH 116/122] firx access to 'Pattern' attribute --- openpype/client/entities.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/openpype/client/entities.py b/openpype/client/entities.py index a9112ac581..43afccf2f1 100644 --- a/openpype/client/entities.py +++ b/openpype/client/entities.py @@ -14,6 +14,8 @@ from bson.objectid import ObjectId from .mongo import get_project_database, get_project_connection +PatternType = type(re.compile("")) + def _prepare_fields(fields, required_fields=None): if not fields: @@ -1054,11 +1056,11 @@ def _regex_filters(filters): for key, value in filters.items(): regexes = [] a_values = [] - if isinstance(value, re.Pattern): + if isinstance(value, PatternType): regexes.append(value) elif isinstance(value, (list, tuple, set)): for item in value: - if isinstance(item, re.Pattern): + if isinstance(item, PatternType): regexes.append(item) else: a_values.append(item) @@ -1194,7 +1196,7 @@ def get_representations( as filter. Filter ignored if 'None' is passed. version_ids (Iterable[str]): Subset ids used as parent filter. Filter ignored if 'None' is passed. - context_filters (Dict[str, List[str, re.Pattern]]): Filter by + context_filters (Dict[str, List[str, PatternType]]): Filter by representation context fields. names_by_version_ids (dict[ObjectId, list[str]]): Complex filtering using version ids and list of names under the version. @@ -1240,7 +1242,7 @@ def get_archived_representations( as filter. Filter ignored if 'None' is passed. version_ids (Iterable[str]): Subset ids used as parent filter. Filter ignored if 'None' is passed. - context_filters (Dict[str, List[str, re.Pattern]]): Filter by + context_filters (Dict[str, List[str, PatternType]]): Filter by representation context fields. names_by_version_ids (dict[ObjectId, List[str]]): Complex filtering using version ids and list of names under the version. From c5ae7e5d6cb704794e444addf93bb9e903795dc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= <33513211+antirotor@users.noreply.github.com> Date: Fri, 9 Sep 2022 17:45:44 +0200 Subject: [PATCH 117/122] Update openpype/hosts/flame/plugins/publish/extract_subset_resources.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- .../hosts/flame/plugins/publish/extract_subset_resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/hosts/flame/plugins/publish/extract_subset_resources.py b/openpype/hosts/flame/plugins/publish/extract_subset_resources.py index 1af6b00654..1d42330e23 100644 --- a/openpype/hosts/flame/plugins/publish/extract_subset_resources.py +++ b/openpype/hosts/flame/plugins/publish/extract_subset_resources.py @@ -96,7 +96,7 @@ class ExtractSubsetResources(openpype.api.Extractor): source_start_handles = instance.data["sourceStartH"] source_end_handles = instance.data["sourceEndH"] - # retime if needed + # retime if needed if r_speed != 1.0: if retimed_handles: # handles are retimed From 87cb5e25b82d3ea352b17794e231924bdbb097ed Mon Sep 17 00:00:00 2001 From: OpenPype Date: Sat, 10 Sep 2022 04:17:04 +0000 Subject: [PATCH 118/122] [Automated] Bump version --- CHANGELOG.md | 45 ++++++++++++++++++--------------------------- openpype/version.py | 2 +- 2 files changed, 19 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6754f1e2e3..0ffb6a996b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,29 +1,36 @@ # Changelog -## [3.14.2-nightly.3](https://github.com/pypeclub/OpenPype/tree/HEAD) +## [3.14.2-nightly.4](https://github.com/pypeclub/OpenPype/tree/HEAD) [Full Changelog](https://github.com/pypeclub/OpenPype/compare/3.14.1...HEAD) **🆕 New features** - Nuke: Build workfile by template [\#3763](https://github.com/pypeclub/OpenPype/pull/3763) +- Houdini: Publishing workfiles [\#3697](https://github.com/pypeclub/OpenPype/pull/3697) +- Global: making collect audio plugin global [\#3679](https://github.com/pypeclub/OpenPype/pull/3679) **🚀 Enhancements** +- Flame: Adding Creator's retimed shot and handles switch [\#3826](https://github.com/pypeclub/OpenPype/pull/3826) +- Flame: OpenPype submenu to batch and media manager [\#3825](https://github.com/pypeclub/OpenPype/pull/3825) +- General: Better pixmap scaling [\#3809](https://github.com/pypeclub/OpenPype/pull/3809) - Photoshop: attempt to speed up ExtractImage [\#3793](https://github.com/pypeclub/OpenPype/pull/3793) - SyncServer: Added cli commands for sync server [\#3765](https://github.com/pypeclub/OpenPype/pull/3765) -- Blender: Publisher collect workfile representation [\#3670](https://github.com/pypeclub/OpenPype/pull/3670) -- Maya: move set render settings menu entry [\#3669](https://github.com/pypeclub/OpenPype/pull/3669) -- Scene Inventory: Maya add actions to select from or to scene [\#3659](https://github.com/pypeclub/OpenPype/pull/3659) +- Kitsu: Drop 'entities root' setting. [\#3739](https://github.com/pypeclub/OpenPype/pull/3739) **🐛 Bug fixes** +- General: Fix Pattern access in client code [\#3828](https://github.com/pypeclub/OpenPype/pull/3828) +- Launcher: Skip opening last work file works for groups [\#3822](https://github.com/pypeclub/OpenPype/pull/3822) +- Maya: Publishing data key change [\#3811](https://github.com/pypeclub/OpenPype/pull/3811) +- Igniter: Fix status handling when version is already installed [\#3804](https://github.com/pypeclub/OpenPype/pull/3804) - Resolve: Addon import is Python 2 compatible [\#3798](https://github.com/pypeclub/OpenPype/pull/3798) +- Hiero: retimed clip publishing is working [\#3792](https://github.com/pypeclub/OpenPype/pull/3792) - nuke: validate write node is not failing due wrong type [\#3780](https://github.com/pypeclub/OpenPype/pull/3780) - Fix - changed format of version string in pyproject.toml [\#3777](https://github.com/pypeclub/OpenPype/pull/3777) - Ftrack status fix typo prgoress -\> progress [\#3761](https://github.com/pypeclub/OpenPype/pull/3761) - Fix version resolution [\#3757](https://github.com/pypeclub/OpenPype/pull/3757) -- Maya: `containerise` dont skip empty values [\#3674](https://github.com/pypeclub/OpenPype/pull/3674) **🔀 Refactored code** @@ -33,17 +40,19 @@ - General: Remove unused teshost [\#3773](https://github.com/pypeclub/OpenPype/pull/3773) - General: Copied 'Extractor' plugin to publish pipeline [\#3771](https://github.com/pypeclub/OpenPype/pull/3771) - General: Move queries of asset and representation links [\#3770](https://github.com/pypeclub/OpenPype/pull/3770) +- General: Move create project folders to pipeline [\#3768](https://github.com/pypeclub/OpenPype/pull/3768) - General: Create project function moved to client code [\#3766](https://github.com/pypeclub/OpenPype/pull/3766) -- General: Move delivery logic to pipeline [\#3751](https://github.com/pypeclub/OpenPype/pull/3751) - General: Move hostdirname functionality into host [\#3749](https://github.com/pypeclub/OpenPype/pull/3749) - General: Move publish utils to pipeline [\#3745](https://github.com/pypeclub/OpenPype/pull/3745) - Houdini: Define houdini as addon [\#3735](https://github.com/pypeclub/OpenPype/pull/3735) +- Fusion: Defined fusion as addon [\#3733](https://github.com/pypeclub/OpenPype/pull/3733) - Flame: Defined flame as addon [\#3732](https://github.com/pypeclub/OpenPype/pull/3732) - Resolve: Define resolve as addon [\#3727](https://github.com/pypeclub/OpenPype/pull/3727) **Merged pull requests:** - Standalone Publisher: Ignore empty labels, then still use name like other asset models [\#3779](https://github.com/pypeclub/OpenPype/pull/3779) +- Kitsu - sync\_all\_project - add list ignore\_projects [\#3776](https://github.com/pypeclub/OpenPype/pull/3776) ## [3.14.1](https://github.com/pypeclub/OpenPype/tree/3.14.1) (2022-08-30) @@ -52,23 +61,16 @@ ### 📖 Documentation - Documentation: Few updates [\#3698](https://github.com/pypeclub/OpenPype/pull/3698) -- Documentation: Settings development [\#3660](https://github.com/pypeclub/OpenPype/pull/3660) - -**🆕 New features** - -- Webpublisher:change create flatten image into tri state [\#3678](https://github.com/pypeclub/OpenPype/pull/3678) -- Blender: validators code correction with settings and defaults [\#3662](https://github.com/pypeclub/OpenPype/pull/3662) **🚀 Enhancements** - General: Thumbnail can use project roots [\#3750](https://github.com/pypeclub/OpenPype/pull/3750) +- git: update gitignore [\#3722](https://github.com/pypeclub/OpenPype/pull/3722) - Settings: Remove settings lock on tray exit [\#3720](https://github.com/pypeclub/OpenPype/pull/3720) - General: Added helper getters to modules manager [\#3712](https://github.com/pypeclub/OpenPype/pull/3712) - Unreal: Define unreal as module and use host class [\#3701](https://github.com/pypeclub/OpenPype/pull/3701) - Settings: Lock settings UI session [\#3700](https://github.com/pypeclub/OpenPype/pull/3700) - General: Benevolent context label collector [\#3686](https://github.com/pypeclub/OpenPype/pull/3686) -- Ftrack: Store ftrack entities on hierarchy integration to instances [\#3677](https://github.com/pypeclub/OpenPype/pull/3677) -- Blender: ops refresh manager after process events [\#3663](https://github.com/pypeclub/OpenPype/pull/3663) **🐛 Bug fixes** @@ -82,11 +84,11 @@ - Settings: Fix project overrides save [\#3708](https://github.com/pypeclub/OpenPype/pull/3708) - Workfiles tool: Fix published workfile filtering [\#3704](https://github.com/pypeclub/OpenPype/pull/3704) - PS, AE: Provide default variant value for workfile subset [\#3703](https://github.com/pypeclub/OpenPype/pull/3703) -- Flame: retime is working on clip publishing [\#3684](https://github.com/pypeclub/OpenPype/pull/3684) - Webpublisher: added check for empty context [\#3682](https://github.com/pypeclub/OpenPype/pull/3682) **🔀 Refactored code** +- General: Move delivery logic to pipeline [\#3751](https://github.com/pypeclub/OpenPype/pull/3751) - General: Host addons cleanup [\#3744](https://github.com/pypeclub/OpenPype/pull/3744) - Webpublisher: Webpublisher is used as addon [\#3740](https://github.com/pypeclub/OpenPype/pull/3740) - Photoshop: Defined photoshop as addon [\#3736](https://github.com/pypeclub/OpenPype/pull/3736) @@ -110,7 +112,6 @@ - Hiero: Define hiero as module [\#3717](https://github.com/pypeclub/OpenPype/pull/3717) - Deadline: better logging for DL webservice failures [\#3694](https://github.com/pypeclub/OpenPype/pull/3694) -- Photoshop: resize saved images in ExtractReview for ffmpeg [\#3676](https://github.com/pypeclub/OpenPype/pull/3676) ## [3.14.0](https://github.com/pypeclub/OpenPype/tree/3.14.0) (2022-08-18) @@ -120,21 +121,11 @@ - Ftrack: Addiotional component metadata [\#3685](https://github.com/pypeclub/OpenPype/pull/3685) - Ftrack: Set task status on farm publishing [\#3680](https://github.com/pypeclub/OpenPype/pull/3680) -- Ftrack: Set task status on task creation in integrate hierarchy [\#3675](https://github.com/pypeclub/OpenPype/pull/3675) -- Maya: Disable rendering of all lights for render instances submitted through Deadline. [\#3661](https://github.com/pypeclub/OpenPype/pull/3661) **🐛 Bug fixes** - General: Switch from hero version to versioned works [\#3691](https://github.com/pypeclub/OpenPype/pull/3691) -- General: Fix finding of last version [\#3656](https://github.com/pypeclub/OpenPype/pull/3656) - -**🔀 Refactored code** - -- General: Use client projects getter [\#3673](https://github.com/pypeclub/OpenPype/pull/3673) - -**Merged pull requests:** - -- Deadline: Global job pre load is not Pype 2 compatible [\#3666](https://github.com/pypeclub/OpenPype/pull/3666) +- Flame: retime is working on clip publishing [\#3684](https://github.com/pypeclub/OpenPype/pull/3684) ## [3.13.0](https://github.com/pypeclub/OpenPype/tree/3.13.0) (2022-08-09) diff --git a/openpype/version.py b/openpype/version.py index c042ca2625..142bd51a30 100644 --- a/openpype/version.py +++ b/openpype/version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- """Package declaring Pype version.""" -__version__ = "3.14.2-nightly.3" +__version__ = "3.14.2-nightly.4" From e76ec9e5aff5651070bcc817e9c0b60a9a980ce4 Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Mon, 12 Sep 2022 16:26:35 +0800 Subject: [PATCH 119/122] adding and loading mel workspace within openpype settings --- openpype/hosts/maya/resources/workspace.mel | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 openpype/hosts/maya/resources/workspace.mel diff --git a/openpype/hosts/maya/resources/workspace.mel b/openpype/hosts/maya/resources/workspace.mel deleted file mode 100644 index f7213fa4f6..0000000000 --- a/openpype/hosts/maya/resources/workspace.mel +++ /dev/null @@ -1,11 +0,0 @@ -//Maya 2018 Project Definition - -workspace -fr "shaders" "renderData/shaders"; -workspace -fr "alembicCache" "cache/alembic"; -workspace -fr "mayaAscii" ""; -workspace -fr "mayaBinary" ""; -workspace -fr "renderData" "renderData"; -workspace -fr "fileCache" "cache/nCache"; -workspace -fr "scene" ""; -workspace -fr "sourceImages" "sourceimages"; -workspace -fr "images" "renders"; From c6ad515682944690d15532cd446fae2d8c93a570 Mon Sep 17 00:00:00 2001 From: OpenPype Date: Mon, 12 Sep 2022 09:46:17 +0000 Subject: [PATCH 120/122] [Automated] Bump version --- CHANGELOG.md | 16 ++++++---------- openpype/version.py | 2 +- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ffb6a996b..cccfc2eded 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [3.14.2-nightly.4](https://github.com/pypeclub/OpenPype/tree/HEAD) +## [3.14.2-nightly.5](https://github.com/pypeclub/OpenPype/tree/HEAD) [Full Changelog](https://github.com/pypeclub/OpenPype/compare/3.14.1...HEAD) @@ -8,7 +8,6 @@ - Nuke: Build workfile by template [\#3763](https://github.com/pypeclub/OpenPype/pull/3763) - Houdini: Publishing workfiles [\#3697](https://github.com/pypeclub/OpenPype/pull/3697) -- Global: making collect audio plugin global [\#3679](https://github.com/pypeclub/OpenPype/pull/3679) **🚀 Enhancements** @@ -18,6 +17,7 @@ - Photoshop: attempt to speed up ExtractImage [\#3793](https://github.com/pypeclub/OpenPype/pull/3793) - SyncServer: Added cli commands for sync server [\#3765](https://github.com/pypeclub/OpenPype/pull/3765) - Kitsu: Drop 'entities root' setting. [\#3739](https://github.com/pypeclub/OpenPype/pull/3739) +- git: update gitignore [\#3722](https://github.com/pypeclub/OpenPype/pull/3722) **🐛 Bug fixes** @@ -42,6 +42,8 @@ - General: Move queries of asset and representation links [\#3770](https://github.com/pypeclub/OpenPype/pull/3770) - General: Move create project folders to pipeline [\#3768](https://github.com/pypeclub/OpenPype/pull/3768) - General: Create project function moved to client code [\#3766](https://github.com/pypeclub/OpenPype/pull/3766) +- Maya: Refactor submit deadline to use AbstractSubmitDeadline [\#3759](https://github.com/pypeclub/OpenPype/pull/3759) +- General: Change publish template settings location [\#3755](https://github.com/pypeclub/OpenPype/pull/3755) - General: Move hostdirname functionality into host [\#3749](https://github.com/pypeclub/OpenPype/pull/3749) - General: Move publish utils to pipeline [\#3745](https://github.com/pypeclub/OpenPype/pull/3745) - Houdini: Define houdini as addon [\#3735](https://github.com/pypeclub/OpenPype/pull/3735) @@ -65,12 +67,12 @@ **🚀 Enhancements** - General: Thumbnail can use project roots [\#3750](https://github.com/pypeclub/OpenPype/pull/3750) -- git: update gitignore [\#3722](https://github.com/pypeclub/OpenPype/pull/3722) - Settings: Remove settings lock on tray exit [\#3720](https://github.com/pypeclub/OpenPype/pull/3720) - General: Added helper getters to modules manager [\#3712](https://github.com/pypeclub/OpenPype/pull/3712) - Unreal: Define unreal as module and use host class [\#3701](https://github.com/pypeclub/OpenPype/pull/3701) - Settings: Lock settings UI session [\#3700](https://github.com/pypeclub/OpenPype/pull/3700) - General: Benevolent context label collector [\#3686](https://github.com/pypeclub/OpenPype/pull/3686) +- Ftrack: Addiotional component metadata [\#3685](https://github.com/pypeclub/OpenPype/pull/3685) **🐛 Bug fixes** @@ -84,7 +86,7 @@ - Settings: Fix project overrides save [\#3708](https://github.com/pypeclub/OpenPype/pull/3708) - Workfiles tool: Fix published workfile filtering [\#3704](https://github.com/pypeclub/OpenPype/pull/3704) - PS, AE: Provide default variant value for workfile subset [\#3703](https://github.com/pypeclub/OpenPype/pull/3703) -- Webpublisher: added check for empty context [\#3682](https://github.com/pypeclub/OpenPype/pull/3682) +- Flame: retime is working on clip publishing [\#3684](https://github.com/pypeclub/OpenPype/pull/3684) **🔀 Refactored code** @@ -117,15 +119,9 @@ [Full Changelog](https://github.com/pypeclub/OpenPype/compare/CI/3.14.0-nightly.1...3.14.0) -**🚀 Enhancements** - -- Ftrack: Addiotional component metadata [\#3685](https://github.com/pypeclub/OpenPype/pull/3685) -- Ftrack: Set task status on farm publishing [\#3680](https://github.com/pypeclub/OpenPype/pull/3680) - **🐛 Bug fixes** - General: Switch from hero version to versioned works [\#3691](https://github.com/pypeclub/OpenPype/pull/3691) -- Flame: retime is working on clip publishing [\#3684](https://github.com/pypeclub/OpenPype/pull/3684) ## [3.13.0](https://github.com/pypeclub/OpenPype/tree/3.13.0) (2022-08-09) diff --git a/openpype/version.py b/openpype/version.py index 142bd51a30..c5dc4ee581 100644 --- a/openpype/version.py +++ b/openpype/version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- """Package declaring Pype version.""" -__version__ = "3.14.2-nightly.4" +__version__ = "3.14.2-nightly.5" From 162370e1ad1291cbbf3eca65266c226ccd119aca Mon Sep 17 00:00:00 2001 From: OpenPype Date: Mon, 12 Sep 2022 09:56:59 +0000 Subject: [PATCH 121/122] [Automated] Release --- CHANGELOG.md | 15 +++++++++------ openpype/version.py | 2 +- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cccfc2eded..46bf56f5bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ # Changelog -## [3.14.2-nightly.5](https://github.com/pypeclub/OpenPype/tree/HEAD) +## [3.14.2](https://github.com/pypeclub/OpenPype/tree/3.14.2) (2022-09-12) -[Full Changelog](https://github.com/pypeclub/OpenPype/compare/3.14.1...HEAD) +[Full Changelog](https://github.com/pypeclub/OpenPype/compare/3.14.1...3.14.2) **🆕 New features** @@ -45,10 +45,11 @@ - Maya: Refactor submit deadline to use AbstractSubmitDeadline [\#3759](https://github.com/pypeclub/OpenPype/pull/3759) - General: Change publish template settings location [\#3755](https://github.com/pypeclub/OpenPype/pull/3755) - General: Move hostdirname functionality into host [\#3749](https://github.com/pypeclub/OpenPype/pull/3749) -- General: Move publish utils to pipeline [\#3745](https://github.com/pypeclub/OpenPype/pull/3745) +- Photoshop: Defined photoshop as addon [\#3736](https://github.com/pypeclub/OpenPype/pull/3736) - Houdini: Define houdini as addon [\#3735](https://github.com/pypeclub/OpenPype/pull/3735) - Fusion: Defined fusion as addon [\#3733](https://github.com/pypeclub/OpenPype/pull/3733) - Flame: Defined flame as addon [\#3732](https://github.com/pypeclub/OpenPype/pull/3732) +- Blender: Define blender as module [\#3729](https://github.com/pypeclub/OpenPype/pull/3729) - Resolve: Define resolve as addon [\#3727](https://github.com/pypeclub/OpenPype/pull/3727) **Merged pull requests:** @@ -72,7 +73,6 @@ - Unreal: Define unreal as module and use host class [\#3701](https://github.com/pypeclub/OpenPype/pull/3701) - Settings: Lock settings UI session [\#3700](https://github.com/pypeclub/OpenPype/pull/3700) - General: Benevolent context label collector [\#3686](https://github.com/pypeclub/OpenPype/pull/3686) -- Ftrack: Addiotional component metadata [\#3685](https://github.com/pypeclub/OpenPype/pull/3685) **🐛 Bug fixes** @@ -91,13 +91,12 @@ **🔀 Refactored code** - General: Move delivery logic to pipeline [\#3751](https://github.com/pypeclub/OpenPype/pull/3751) +- General: Move publish utils to pipeline [\#3745](https://github.com/pypeclub/OpenPype/pull/3745) - General: Host addons cleanup [\#3744](https://github.com/pypeclub/OpenPype/pull/3744) - Webpublisher: Webpublisher is used as addon [\#3740](https://github.com/pypeclub/OpenPype/pull/3740) -- Photoshop: Defined photoshop as addon [\#3736](https://github.com/pypeclub/OpenPype/pull/3736) - Harmony: Defined harmony as addon [\#3734](https://github.com/pypeclub/OpenPype/pull/3734) - General: Module interfaces cleanup [\#3731](https://github.com/pypeclub/OpenPype/pull/3731) - AfterEffects: Move AE functions from general lib [\#3730](https://github.com/pypeclub/OpenPype/pull/3730) -- Blender: Define blender as module [\#3729](https://github.com/pypeclub/OpenPype/pull/3729) - AfterEffects: Define AfterEffects as module [\#3728](https://github.com/pypeclub/OpenPype/pull/3728) - General: Replace PypeLogger with Logger [\#3725](https://github.com/pypeclub/OpenPype/pull/3725) - Nuke: Define nuke as module [\#3724](https://github.com/pypeclub/OpenPype/pull/3724) @@ -119,6 +118,10 @@ [Full Changelog](https://github.com/pypeclub/OpenPype/compare/CI/3.14.0-nightly.1...3.14.0) +**🚀 Enhancements** + +- Ftrack: Addiotional component metadata [\#3685](https://github.com/pypeclub/OpenPype/pull/3685) + **🐛 Bug fixes** - General: Switch from hero version to versioned works [\#3691](https://github.com/pypeclub/OpenPype/pull/3691) diff --git a/openpype/version.py b/openpype/version.py index c5dc4ee581..8469b1712a 100644 --- a/openpype/version.py +++ b/openpype/version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- """Package declaring Pype version.""" -__version__ = "3.14.2-nightly.5" +__version__ = "3.14.2" From 6e2ffc1e5ceb134f17fcedd1646f2cec0014a43a Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 13 Sep 2022 10:33:41 +0200 Subject: [PATCH 122/122] Remove getting project name and settings twice --- openpype/hosts/maya/api/pipeline.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/openpype/hosts/maya/api/pipeline.py b/openpype/hosts/maya/api/pipeline.py index acd8a55aa4..45c52cd0d5 100644 --- a/openpype/hosts/maya/api/pipeline.py +++ b/openpype/hosts/maya/api/pipeline.py @@ -66,8 +66,6 @@ class MayaHost(HostBase, IWorkfileHost, ILoadHost): project_name = legacy_io.active_project() project_settings = get_project_settings(project_name) # process path mapping - project_name = legacy_io.active_project() - project_settings = get_project_settings(project_name) dirmap_processor = MayaDirmap("maya", project_name, project_settings) dirmap_processor.process_dirmap()