From 44757580b78970566fb8e3a5ac3871425dacf408 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Fri, 9 Sep 2022 15:08:10 +0200 Subject: [PATCH 01/12] Fix very slow `get_container_members` calls for instances --- openpype/hosts/maya/api/lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/hosts/maya/api/lib.py b/openpype/hosts/maya/api/lib.py index 58e160cb2f..06faa123f5 100644 --- a/openpype/hosts/maya/api/lib.py +++ b/openpype/hosts/maya/api/lib.py @@ -1532,7 +1532,7 @@ def get_container_members(container): if ref.rsplit(":", 1)[-1].startswith("_UNKNOWN_REF_NODE_"): continue - reference_members = cmds.referenceQuery(ref, nodes=True) + reference_members = cmds.referenceQuery(ref, nodes=True, dagPath=True) reference_members = cmds.ls(reference_members, long=True, objectsOnly=True) From 645971c07e3df0d4a5dc1cdc564f5d147f592f56 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 31 Oct 2022 19:04:27 +0100 Subject: [PATCH 02/12] better deffer reset of publisher --- openpype/tools/publisher/window.py | 40 ++++++++++++++++++------------ 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/openpype/tools/publisher/window.py b/openpype/tools/publisher/window.py index a3387043b8..d8a69bbeb0 100644 --- a/openpype/tools/publisher/window.py +++ b/openpype/tools/publisher/window.py @@ -225,6 +225,12 @@ class PublisherWindow(QtWidgets.QDialog): # Floating publish frame publish_frame = PublishFrame(controller, self.footer_border, self) + # Timer started on show -> connected to timer counter + # - helps to deffer on show logic by 3 event loops + show_timer = QtCore.QTimer() + show_timer.setInterval(1) + show_timer.timeout.connect(self._on_show_timer) + errors_dialog_message_timer = QtCore.QTimer() errors_dialog_message_timer.setInterval(100) errors_dialog_message_timer.timeout.connect( @@ -329,7 +335,6 @@ class PublisherWindow(QtWidgets.QDialog): # forin init self._reset_on_first_show = reset_on_show self._reset_on_show = True - self._restart_timer = None self._publish_frame_visible = None self._error_messages_to_show = collections.deque() @@ -337,6 +342,9 @@ class PublisherWindow(QtWidgets.QDialog): self._set_publish_visibility(False) + self._show_timer = show_timer + self._show_counter = 0 + @property def controller(self): return self._controller @@ -347,17 +355,7 @@ class PublisherWindow(QtWidgets.QDialog): self._first_show = False self._on_first_show() - if not self._reset_on_show: - return - - self._reset_on_show = False - # Detach showing - give OS chance to draw the window - timer = QtCore.QTimer() - timer.setSingleShot(True) - timer.setInterval(1) - timer.timeout.connect(self._on_show_restart_timer) - self._restart_timer = timer - timer.start() + self._show_timer.start() def resizeEvent(self, event): super(PublisherWindow, self).resizeEvent(event) @@ -374,11 +372,21 @@ class PublisherWindow(QtWidgets.QDialog): self.setStyleSheet(style.load_stylesheet()) self._reset_on_show = self._reset_on_first_show - def _on_show_restart_timer(self): - """Callback for '_restart_timer' timer.""" + def _on_show_timer(self): + # Add 1 to counter until hits 2 + if self._show_counter < 3: + self._show_counter += 1 + return - self._restart_timer = None - self.reset() + # Stop the timer + self._show_timer.stop() + # Reset counter when done for next show event + self._show_counter = 0 + + # Reset if requested + if self._reset_on_show: + self._reset_on_show = False + self.reset() def closeEvent(self, event): self.save_changes() From 638f8238250cdd4ce7935c9c637dd0c3e80746ec Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Tue, 1 Nov 2022 12:07:03 +0100 Subject: [PATCH 03/12] run openpype subprocess using 'Popen' instead of 'check_output' function --- .../repository/custom/plugins/GlobalJobPreLoad.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/openpype/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py b/openpype/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py index 61b95cf06d..b8d90cdf69 100644 --- a/openpype/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py +++ b/openpype/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py @@ -193,10 +193,17 @@ def inject_openpype_environment(deadlinePlugin): env["AVALON_TIMEOUT"] = "5000" print(">>> Executing: {}".format(" ".join(args))) - std_output = subprocess.check_output(args, - cwd=os.path.dirname(exe), - env=env) - print(">>> Process result {}".format(std_output)) + proc = subprocess.Popen( + args, + cwd=os.path.dirname(exe), + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + std_output, std_err = proc.communicate() + print(">>> Process result {}\n".format(std_output, std_err)) + if proc.returncode != 0: + raise RuntimeError("OpenPype process failed.") print(">>> Loading file ...") with open(export_url) as fp: From 298fbcae701f2bcc2833e8b5c4310c40c05fe380 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Tue, 1 Nov 2022 12:07:30 +0100 Subject: [PATCH 04/12] removed unnecessary headless environment (handled by '--headless' arg) --- .../deadline/repository/custom/plugins/GlobalJobPreLoad.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openpype/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py b/openpype/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py index b8d90cdf69..a25a1b7e93 100644 --- a/openpype/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py +++ b/openpype/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py @@ -189,7 +189,6 @@ def inject_openpype_environment(deadlinePlugin): print(">>> Missing OPENPYPE_MONGO env var, process won't work") env = os.environ - env["OPENPYPE_HEADLESS_MODE"] = "1" env["AVALON_TIMEOUT"] = "5000" print(">>> Executing: {}".format(" ".join(args))) From c178d6e2a1f9251ba9cfb09d1ad62cf62a30e411 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Tue, 1 Nov 2022 12:07:37 +0100 Subject: [PATCH 05/12] formatting changes --- .../deadline/repository/custom/plugins/GlobalJobPreLoad.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py b/openpype/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py index a25a1b7e93..80f91607bc 100644 --- a/openpype/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py +++ b/openpype/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py @@ -164,7 +164,7 @@ def inject_openpype_environment(deadlinePlugin): args = [ exe, "--headless", - 'extractenvironments', + "extractenvironments", export_url ] From 4161b3b48db137a7bdd1b998612ea6450cc3ea54 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Tue, 1 Nov 2022 12:56:53 +0100 Subject: [PATCH 06/12] use Deadline 'ProcessUtils' to run openpype process --- .../custom/plugins/GlobalJobPreLoad.py | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/openpype/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py b/openpype/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py index 80f91607bc..61d0c8eb86 100644 --- a/openpype/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py +++ b/openpype/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py @@ -7,7 +7,12 @@ import json import platform import uuid import re -from Deadline.Scripting import RepositoryUtils, FileUtils, DirectoryUtils +from Deadline.Scripting import ( + RepositoryUtils, + FileUtils, + DirectoryUtils, + ProcessUtils, +) def get_openpype_version_from_path(path, build=True): @@ -162,7 +167,6 @@ def inject_openpype_environment(deadlinePlugin): print(">>> Temporary path: {}".format(export_url)) args = [ - exe, "--headless", "extractenvironments", export_url @@ -188,21 +192,16 @@ def inject_openpype_environment(deadlinePlugin): if not os.environ.get("OPENPYPE_MONGO"): print(">>> Missing OPENPYPE_MONGO env var, process won't work") - env = os.environ - env["AVALON_TIMEOUT"] = "5000" + os.environ["AVALON_TIMEOUT"] = "5000" - print(">>> Executing: {}".format(" ".join(args))) - proc = subprocess.Popen( - args, - cwd=os.path.dirname(exe), - env=env, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - std_output, std_err = proc.communicate() - print(">>> Process result {}\n".format(std_output, std_err)) - if proc.returncode != 0: - raise RuntimeError("OpenPype process failed.") + args_str = subprocess.list2cmdline(args) + print(">>> Executing: {} {}".format(exe, args_str)) + process = ProcessUtils.SpawnProcess(exe, args_str, os.path.dirname(exe)) + ProcessUtils.WaitForExit(process, -1) + if process.ExitCode != 0: + raise RuntimeError( + "Failed to run OpenPype process to extract environments." + ) print(">>> Loading file ...") with open(export_url) as fp: From f1e198ea94b0c13d90a7419e2b5d3a4dc1769b60 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Tue, 1 Nov 2022 12:58:22 +0100 Subject: [PATCH 07/12] fix too long line --- .../deadline/repository/custom/plugins/GlobalJobPreLoad.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/openpype/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py b/openpype/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py index 61d0c8eb86..9b35c9502d 100644 --- a/openpype/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py +++ b/openpype/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py @@ -196,7 +196,9 @@ def inject_openpype_environment(deadlinePlugin): args_str = subprocess.list2cmdline(args) print(">>> Executing: {} {}".format(exe, args_str)) - process = ProcessUtils.SpawnProcess(exe, args_str, os.path.dirname(exe)) + process = ProcessUtils.SpawnProcess( + exe, args_str, os.path.dirname(exe) + ) ProcessUtils.WaitForExit(process, -1) if process.ExitCode != 0: raise RuntimeError( From 761d624b2e40901be713068157ad7d7aeaeb3bb4 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Tue, 1 Nov 2022 16:07:50 +0100 Subject: [PATCH 08/12] fix comparison of repre name --- openpype/tools/loader/widgets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/tools/loader/widgets.py b/openpype/tools/loader/widgets.py index d37ce500e0..826c7110da 100644 --- a/openpype/tools/loader/widgets.py +++ b/openpype/tools/loader/widgets.py @@ -515,7 +515,7 @@ class SubsetWidget(QtWidgets.QWidget): if not one_item_selected: # Filter loaders from first subset by intersected combinations for repre, loader in first_loaders: - if (repre["name"], loader) not in found_combinations: + if (repre["name"].lower(), loader) not in found_combinations: continue loaders.append((repre, loader)) From bae7a9960a4f28e23aab10f959789eaa00c20c87 Mon Sep 17 00:00:00 2001 From: OpenPype Date: Wed, 2 Nov 2022 04:02:57 +0000 Subject: [PATCH 09/12] [Automated] Bump version --- openpype/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/version.py b/openpype/version.py index 442c5f033b..46bb4b1cd0 100644 --- a/openpype/version.py +++ b/openpype/version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- """Package declaring Pype version.""" -__version__ = "3.14.7-nightly.1" +__version__ = "3.14.7-nightly.2" From e5205f5c81b677209e1626866c100ffd169ac414 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 2 Nov 2022 19:53:03 +0100 Subject: [PATCH 10/12] prepared common function to cache instances during collection phase --- openpype/pipeline/create/__init__.py | 2 ++ openpype/pipeline/create/creator_plugins.py | 32 +++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/openpype/pipeline/create/__init__.py b/openpype/pipeline/create/__init__.py index 4b91951a08..9e858151fd 100644 --- a/openpype/pipeline/create/__init__.py +++ b/openpype/pipeline/create/__init__.py @@ -24,6 +24,8 @@ from .creator_plugins import ( deregister_creator_plugin, register_creator_plugin_path, deregister_creator_plugin_path, + + cache_and_get_instances, ) from .context import ( diff --git a/openpype/pipeline/create/creator_plugins.py b/openpype/pipeline/create/creator_plugins.py index c69abb8861..0f9c346966 100644 --- a/openpype/pipeline/create/creator_plugins.py +++ b/openpype/pipeline/create/creator_plugins.py @@ -1,5 +1,6 @@ import os import copy +import collections from abc import ( ABCMeta, @@ -660,3 +661,34 @@ def deregister_creator_plugin_path(path): deregister_plugin_path(BaseCreator, path) deregister_plugin_path(LegacyCreator, path) deregister_plugin_path(SubsetConvertorPlugin, path) + + +def cache_and_get_instances(creator, shared_key, list_instances_func): + """Common approach to cache instances in shared data. + + This is helper function which does not handle cases when a 'shared_key' is + used for different list instances functions. The same approach of caching + instances into 'collection_shared_data' is not required but is so common + we've decided to unify it to some degree. + + Function 'list_instances_func' is called only if 'shared_key' is not + available in 'collection_shared_data' on creator. + + Args: + creator (Creator): Plugin which would like to get instance data. + shared_key (str): Key under which output of function will be stored. + list_instances_func (Function): Function that will return instance data + if data were not yet stored under 'shared_key'. + + Returns: + Dict[str, Dict[str, Any]]: Cached instances by creator identifier from + result of passed function. + """ + + if shared_key not in creator.collection_shared_data: + value = collections.defaultdict(list) + for instance in list_instances_func(): + identifier = instance.get("creator_identifier") + value[identifier].append(instance) + creator.collection_shared_data[shared_key] = value + return creator.collection_shared_data[shared_key] From 53467f97f941eaeb60651f177c5639d4b45f314d Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 2 Nov 2022 19:54:01 +0100 Subject: [PATCH 11/12] use new function in tray publisher --- openpype/hosts/traypublisher/api/plugin.py | 45 ++++++++-------------- 1 file changed, 15 insertions(+), 30 deletions(-) diff --git a/openpype/hosts/traypublisher/api/plugin.py b/openpype/hosts/traypublisher/api/plugin.py index 555041d389..24d7004e84 100644 --- a/openpype/hosts/traypublisher/api/plugin.py +++ b/openpype/hosts/traypublisher/api/plugin.py @@ -2,7 +2,8 @@ from openpype.lib.attribute_definitions import FileDef from openpype.pipeline.create import ( Creator, HiddenCreator, - CreatedInstance + CreatedInstance, + cache_and_get_instances, ) from .pipeline import ( @@ -16,34 +17,19 @@ from openpype.lib.transcoding import IMAGE_EXTENSIONS, VIDEO_EXTENSIONS REVIEW_EXTENSIONS = IMAGE_EXTENSIONS + VIDEO_EXTENSIONS - -def _cache_and_get_instances(creator): - """Cache instances in shared data. - - Args: - creator (Creator): Plugin which would like to get instances from host. - - Returns: - List[Dict[str, Any]]: Cached instances list from host implementation. - """ - - shared_key = "openpype.traypublisher.instances" - if shared_key not in creator.collection_shared_data: - creator.collection_shared_data[shared_key] = list_instances() - return creator.collection_shared_data[shared_key] +SHARED_DATA_KEY = "openpype.traypublisher.instances" class HiddenTrayPublishCreator(HiddenCreator): host_name = "traypublisher" def collect_instances(self): - for instance_data in _cache_and_get_instances(self): - creator_id = instance_data.get("creator_identifier") - if creator_id == self.identifier: - instance = CreatedInstance.from_existing( - instance_data, self - ) - self._add_instance_to_context(instance) + instances_by_identifier = cache_and_get_instances( + self, SHARED_DATA_KEY, list_instances + ) + for instance_data in instances_by_identifier[self.identifier]: + instance = CreatedInstance.from_existing(instance_data, self) + self._add_instance_to_context(instance) def update_instances(self, update_list): update_instances(update_list) @@ -74,13 +60,12 @@ class TrayPublishCreator(Creator): host_name = "traypublisher" def collect_instances(self): - for instance_data in _cache_and_get_instances(self): - creator_id = instance_data.get("creator_identifier") - if creator_id == self.identifier: - instance = CreatedInstance.from_existing( - instance_data, self - ) - self._add_instance_to_context(instance) + instances_by_identifier = cache_and_get_instances( + self, SHARED_DATA_KEY, list_instances + ) + for instance_data in instances_by_identifier[self.identifier]: + instance = CreatedInstance.from_existing(instance_data, self) + self._add_instance_to_context(instance) def update_instances(self, update_list): update_instances(update_list) From eaa097d513ba5ec55e9b80267f634ca77d39fe9b Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 2 Nov 2022 19:54:13 +0100 Subject: [PATCH 12/12] change 'REVIEW_EXTENSIONS' to set instead of list --- openpype/hosts/traypublisher/api/plugin.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/openpype/hosts/traypublisher/api/plugin.py b/openpype/hosts/traypublisher/api/plugin.py index 24d7004e84..6b95379cf2 100644 --- a/openpype/hosts/traypublisher/api/plugin.py +++ b/openpype/hosts/traypublisher/api/plugin.py @@ -1,22 +1,19 @@ from openpype.lib.attribute_definitions import FileDef +from openpype.lib.transcoding import IMAGE_EXTENSIONS, VIDEO_EXTENSIONS from openpype.pipeline.create import ( Creator, HiddenCreator, CreatedInstance, cache_and_get_instances, ) - from .pipeline import ( list_instances, update_instances, remove_instances, HostContext, ) -from openpype.lib.transcoding import IMAGE_EXTENSIONS, VIDEO_EXTENSIONS - - -REVIEW_EXTENSIONS = IMAGE_EXTENSIONS + VIDEO_EXTENSIONS +REVIEW_EXTENSIONS = set(IMAGE_EXTENSIONS) | set(VIDEO_EXTENSIONS) SHARED_DATA_KEY = "openpype.traypublisher.instances"