From bacbff0262e8306ef16ccf3d1e826e3e1cb2728f Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 17 Oct 2022 15:19:48 +0200 Subject: [PATCH 01/10] create context has callbacks for reset preparation and finalization --- openpype/pipeline/create/context.py | 15 +++++++++++++++ openpype/tools/publisher/control.py | 4 ++++ 2 files changed, 19 insertions(+) diff --git a/openpype/pipeline/create/context.py b/openpype/pipeline/create/context.py index c1cf4dab44..1f3c32f0a7 100644 --- a/openpype/pipeline/create/context.py +++ b/openpype/pipeline/create/context.py @@ -860,6 +860,9 @@ class CreateContext: All changes will be lost if were not saved explicitely. """ + + self.reset_preparation() + self.reset_avalon_context() self.reset_plugins(discover_publish_plugins) self.reset_context_data() @@ -868,6 +871,18 @@ class CreateContext: self.reset_instances() self.execute_autocreators() + self.reset_finalization() + + def reset_preparation(self): + """Prepare attributes that must be prepared/cleaned before reset.""" + + pass + + def reset_finalization(self): + """Cleanup of attributes after reset.""" + + pass + def reset_avalon_context(self): """Give ability to reset avalon context. diff --git a/openpype/tools/publisher/control.py b/openpype/tools/publisher/control.py index b4c89f221f..19e28cca4b 100644 --- a/openpype/tools/publisher/control.py +++ b/openpype/tools/publisher/control.py @@ -552,6 +552,8 @@ class PublisherController: self.save_changes() + self.create_context.reset_preparation() + # Reset avalon context self.create_context.reset_avalon_context() @@ -560,6 +562,8 @@ class PublisherController: self._reset_publish() self._reset_instances() + self.create_context.reset_finalization() + self.emit_card_message("Refreshed..") def _reset_plugins(self): From e0bb8c0469d50273ad041280b786380d98de080c Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 17 Oct 2022 15:20:06 +0200 Subject: [PATCH 02/10] context can handle shared data for collection phase --- openpype/pipeline/create/context.py | 68 ++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/openpype/pipeline/create/context.py b/openpype/pipeline/create/context.py index 1f3c32f0a7..02398818d9 100644 --- a/openpype/pipeline/create/context.py +++ b/openpype/pipeline/create/context.py @@ -27,6 +27,11 @@ from .creator_plugins import ( UpdateData = collections.namedtuple("UpdateData", ["instance", "changes"]) +class UnavailableSharedData(Exception): + """Shared data are not available at the moment when are accessed.""" + pass + + class ImmutableKeyError(TypeError): """Accessed key is immutable so does not allow changes or removements.""" @@ -809,6 +814,9 @@ class CreateContext: self._bulk_counter = 0 self._bulk_instances_to_process = [] + # Shared data across creators during collection phase + self._collection_shared_data = None + # Trigger reset if was enabled if reset: self.reset(discover_publish_plugins) @@ -877,11 +885,15 @@ class CreateContext: """Prepare attributes that must be prepared/cleaned before reset.""" pass + # Give ability to store shared data for collection phase + self._collection_shared_data = {} def reset_finalization(self): """Cleanup of attributes after reset.""" pass + # Stop access to collection shared data + self._collection_shared_data = None def reset_avalon_context(self): """Give ability to reset avalon context. @@ -991,7 +1003,8 @@ class CreateContext: and creator_class.host_name != self.host_name ): self.log.info(( - "Creator's host name is not supported for current host {}" + "Creator's host name \"{}\"" + " is not supported for current host \"{}\"" ).format(creator_class.host_name, self.host_name)) continue @@ -1266,3 +1279,56 @@ class CreateContext: if not plugin.__instanceEnabled__: plugins.append(plugin) return plugins + + def _validate_collection_shared_data(self): + if self._collection_shared_data is None: + raise UnavailableSharedData( + "Accessed Collection shared data out of collection phase" + ) + + def has_collection_shared_data(self, key): + """Check if collection shared data are set. + + Args: + key (str): Key under which are shared data stored. + + Retruns: + bool: Key is already set. + + Raises: + UnavailableSharedData: When called out of collection phase. + """ + + self._validate_collection_shared_data() + return key in self._collection_shared_data + + def get_collection_shared_data(self, key, default=None): + """Receive shared data during collection phase. + + Args: + key (str): Key under which are shared data stored. + default (Any): Default value if key is not set. + + Returns: + Any: Value stored under the key. + + Raises: + UnavailableSharedData: When called out of collection phase. + """ + + self._validate_collection_shared_data() + return self._collection_shared_data.get(key, default) + + def set_collection_shared_data(self, key, value): + """Store a value under collection shared data. + + Args: + key (str): Key under which will shared data be stored. + value (Any): Value to store. + + Raises: + UnavailableSharedData: When called out of collection phase. + """ + + self._validate_collection_shared_data() + self._collection_shared_data[key] = value From 2ed383c4768571436df3f2b3b2245d55ccdfdc6b Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 17 Oct 2022 15:20:24 +0200 Subject: [PATCH 03/10] added wrappers for access to shared data in create plugins --- openpype/pipeline/create/creator_plugins.py | 36 +++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/openpype/pipeline/create/creator_plugins.py b/openpype/pipeline/create/creator_plugins.py index 05ba8902aa..761054fbd5 100644 --- a/openpype/pipeline/create/creator_plugins.py +++ b/openpype/pipeline/create/creator_plugins.py @@ -6,6 +6,7 @@ from abc import ( abstractmethod, abstractproperty ) + import six from openpype.settings import get_system_settings, get_project_settings @@ -323,6 +324,41 @@ class BaseCreator: return self.instance_attr_defs + def has_collection_shared_data(self, key): + """Check if collection shared data are set. + + Args: + key (str): Key under which are shared data stored. + + Retruns: + bool: Key is already set. + """ + + return self.create_context.has_collection_shared_data(key) + + def get_collection_shared_data(self, key, default=None): + """Receive shared data during collection phase. + + Args: + key (str): Key under which are shared data stored. + default (Any): Default value if key is not set. + + Returns: + Any: Value stored under the key. + """ + + return self.create_context.get_collection_shared_data(key, default) + + def set_collection_shared_data(self, key, value): + """Store a value under collection shared data. + + Args: + key (str): Key under which will shared data be stored. + value (Any): Value to store. + """ + + return self.create_context.set_collection_shared_data(key, value) + class Creator(BaseCreator): """Creator that has more information for artist to show in UI. From 3f01d008c59205136d18675068f4242def604b2f Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 17 Oct 2022 15:26:38 +0200 Subject: [PATCH 04/10] added recommendation --- openpype/pipeline/create/context.py | 3 +++ openpype/pipeline/create/creator_plugins.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/openpype/pipeline/create/context.py b/openpype/pipeline/create/context.py index 02398818d9..613eaa2865 100644 --- a/openpype/pipeline/create/context.py +++ b/openpype/pipeline/create/context.py @@ -1322,6 +1322,9 @@ class CreateContext: def set_collection_shared_data(self, key, value): """Store a value under collection shared data. + It is highly recommended to use very specific keys as creators may + clash each other if simple keys are used. + Args: key (str): Key under which will shared data be stored. value (Any): Value to store. diff --git a/openpype/pipeline/create/creator_plugins.py b/openpype/pipeline/create/creator_plugins.py index 761054fbd5..343a416872 100644 --- a/openpype/pipeline/create/creator_plugins.py +++ b/openpype/pipeline/create/creator_plugins.py @@ -352,6 +352,9 @@ class BaseCreator: def set_collection_shared_data(self, key, value): """Store a value under collection shared data. + It is highly recommended to use very specific keys as creators may + clash each other if simple keys are used. + Args: key (str): Key under which will shared data be stored. value (Any): Value to store. From 0aefb39acb85b4f8dc0de3904b3613c3762d8c2c Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 17 Oct 2022 15:31:07 +0200 Subject: [PATCH 05/10] cache instances in shared data in tray publisher --- openpype/hosts/traypublisher/api/plugin.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/traypublisher/api/plugin.py b/openpype/hosts/traypublisher/api/plugin.py index 89c25389cb..1e592e786d 100644 --- a/openpype/hosts/traypublisher/api/plugin.py +++ b/openpype/hosts/traypublisher/api/plugin.py @@ -17,11 +17,27 @@ from openpype.lib.transcoding import IMAGE_EXTENSIONS, VIDEO_EXTENSIONS REVIEW_EXTENSIONS = IMAGE_EXTENSIONS + VIDEO_EXTENSIONS +def _cache_and_get_instances(creator): + """Cache instances in shared data. + + Args: + creator (Creator): Plugin which would like to get instances from host. + + Returns: + List[Dict[str, Any]]: Cached instances list from host implementation. + """ + + shared_key = "openpype.traypublisher.instances" + if not creator.has_collection_shared_data(shared_key): + creator.set_collection_shared_data(shared_key, list_instances()) + return creator.get_collection_shared_data(shared_key) + + class HiddenTrayPublishCreator(HiddenCreator): host_name = "traypublisher" def collect_instances(self): - for instance_data in list_instances(): + for instance_data in _cache_and_get_instances(): creator_id = instance_data.get("creator_identifier") if creator_id == self.identifier: instance = CreatedInstance.from_existing( @@ -58,7 +74,7 @@ class TrayPublishCreator(Creator): host_name = "traypublisher" def collect_instances(self): - for instance_data in list_instances(): + for instance_data in _cache_and_get_instances(): creator_id = instance_data.get("creator_identifier") if creator_id == self.identifier: instance = CreatedInstance.from_existing( From aa6de1cfeba7338212c4043a3674fe50f6ecab90 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 17 Oct 2022 16:51:15 +0200 Subject: [PATCH 06/10] renamed 'has_collection_shared_data' to 'collection_shared_data_contains' --- openpype/hosts/traypublisher/api/plugin.py | 2 +- openpype/pipeline/create/context.py | 2 +- openpype/pipeline/create/creator_plugins.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/openpype/hosts/traypublisher/api/plugin.py b/openpype/hosts/traypublisher/api/plugin.py index 1e592e786d..0f519e3c32 100644 --- a/openpype/hosts/traypublisher/api/plugin.py +++ b/openpype/hosts/traypublisher/api/plugin.py @@ -28,7 +28,7 @@ def _cache_and_get_instances(creator): """ shared_key = "openpype.traypublisher.instances" - if not creator.has_collection_shared_data(shared_key): + if not creator.collection_shared_data_contains(shared_key): creator.set_collection_shared_data(shared_key, list_instances()) return creator.get_collection_shared_data(shared_key) diff --git a/openpype/pipeline/create/context.py b/openpype/pipeline/create/context.py index 613eaa2865..298eacecb5 100644 --- a/openpype/pipeline/create/context.py +++ b/openpype/pipeline/create/context.py @@ -1286,7 +1286,7 @@ class CreateContext: "Accessed Collection shared data out of collection phase" ) - def has_collection_shared_data(self, key): + def collection_shared_data_contains(self, key): """Check if collection shared data are set. Args: diff --git a/openpype/pipeline/create/creator_plugins.py b/openpype/pipeline/create/creator_plugins.py index 343a416872..e5018c395e 100644 --- a/openpype/pipeline/create/creator_plugins.py +++ b/openpype/pipeline/create/creator_plugins.py @@ -324,7 +324,7 @@ class BaseCreator: return self.instance_attr_defs - def has_collection_shared_data(self, key): + def collection_shared_data_contains(self, key): """Check if collection shared data are set. Args: @@ -334,7 +334,7 @@ class BaseCreator: bool: Key is already set. """ - return self.create_context.has_collection_shared_data(key) + return self.create_context.collection_shared_data_contains(key) def get_collection_shared_data(self, key, default=None): """Receive shared data during collection phase. From ba2cb2d11d7dbc0384265071a9d464b68a8813f8 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 17 Oct 2022 17:00:44 +0200 Subject: [PATCH 07/10] add information about shared data to documentation --- website/docs/dev_publishing.md | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/website/docs/dev_publishing.md b/website/docs/dev_publishing.md index 7a6082a517..5f30f7f9c8 100644 --- a/website/docs/dev_publishing.md +++ b/website/docs/dev_publishing.md @@ -47,10 +47,14 @@ Context discovers creator and publish plugins. Trigger collections of existing i Creator plugins can call **creator_adds_instance** or **creator_removed_instance** to add/remove instances but these methods are not meant to be called directly out of the creator. The reason is that it is the creator's responsibility to remove metadata or decide if it should remove the instance. -#### Required functions in host implementation -Host implementation **must** implement **get_context_data** and **update_context_data**. These two functions are needed to store metadata that are not related to any instance but are needed for Creating and publishing process. Right now only data about enabled/disabled optional publish plugins is stored there. When data is not stored and loaded properly, reset of publishing will cause that they will be set to default value. Context data also parsed to json string similarly as instance data. +During reset are re-cached Creator plugins, re-collected instances, refreshed host context and more. Object of `CreateContext` supply shared data during the reset. They can be used by creators to share same data needed during collection phase or during creation for autocreators. -There are also few optional functions. For UI purposes it is possible to implement **get_context_title** which can return a string shown in UI as a title. Output string may contain html tags. It is recommended to return context path (it will be created function this purposes) in this order `"{project name}/{asset hierarchy}/{asset name}/{task name}"`. +#### Required functions in host implementation +It is recommended to use `HostBase` class (`from openpype.host import HostBase`) as base for host implementation with combination of `IPublishHost` interface (`from openpype.host import IPublishHost`). These abstract classes should guide you to fill missing attributes and methods. + +To sum them and in case host implementation is inheriting `HostBase` the implementation **must** implement **get_context_data** and **update_context_data**. These two functions are needed to store metadata that are not related to any instance but are needed for Creating and publishing process. Right now only data about enabled/disabled optional publish plugins is stored there. When data is not stored and loaded properly, reset of publishing will cause that they will be set to default value. Context data also parsed to json string similarly as instance data. + +There are also few optional functions. For UI purposes it is possible to implement **get_context_title** which can return a string shown in UI as a title. Output string may contain html tags. It is recommended to return context path (it will be created function this purposes) in this order `"{project name}/{asset hierarchy}/{asset name}/{task name}"` (this is default implementation in `HostBase`). Another optional function is **get_current_context**. This function is handy in hosts where it is possible to open multiple workfiles in one process so using global context variables is not relevant because artists can switch between opened workfiles without being acknowledged. When a function is not implemented or won't return the right keys the global context is used. ```json @@ -68,6 +72,12 @@ Main responsibility of create plugin is to create, update, collect and remove in #### *BaseCreator* Base implementation of creator plugin. It is not recommended to use this class as base for production plugins but rather use one of **HiddenCreator**, **AutoCreator** and **Creator** variants. +**Access to shared data** +Functions to work with "Collection shared data" can be used during reset phase of `CreateContext`. Creators can cache there data that are common for them. For example list of nodes in scene. Methods are implemented on `CreateContext` but their usage is primarily for Create plugins as nothing else should use it. +- **`collection_shared_data_contains`** - Check if shared data already has set a key. +- **`get_collection_shared_data`** - Receive value of shared data by a key. +- **`set_collection_shared_data`** - Set or update value of shared data key. + **Abstractions** - **`family`** (class attr) - Tells what kind of instance will be created. ```python From cad97d6d1dc98d9d2d46dec060801fa4645e6053 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Tue, 18 Oct 2022 15:18:24 +0200 Subject: [PATCH 08/10] simplify api by giving access to 'collection_shared_data' property --- openpype/hosts/traypublisher/api/plugin.py | 6 +- openpype/pipeline/create/context.py | 65 ++++----------------- openpype/pipeline/create/creator_plugins.py | 41 +++---------- 3 files changed, 23 insertions(+), 89 deletions(-) diff --git a/openpype/hosts/traypublisher/api/plugin.py b/openpype/hosts/traypublisher/api/plugin.py index 0f519e3c32..2cb5a8729f 100644 --- a/openpype/hosts/traypublisher/api/plugin.py +++ b/openpype/hosts/traypublisher/api/plugin.py @@ -28,9 +28,9 @@ def _cache_and_get_instances(creator): """ shared_key = "openpype.traypublisher.instances" - if not creator.collection_shared_data_contains(shared_key): - creator.set_collection_shared_data(shared_key, list_instances()) - return creator.get_collection_shared_data(shared_key) + if shared_key not in creator.collection_shared_data: + creator.collection_shared_data[shared_key] = list_instances() + return creator.collection_shared_data[shared_key] class HiddenTrayPublishCreator(HiddenCreator): diff --git a/openpype/pipeline/create/context.py b/openpype/pipeline/create/context.py index 298eacecb5..c5c9a14f33 100644 --- a/openpype/pipeline/create/context.py +++ b/openpype/pipeline/create/context.py @@ -884,14 +884,12 @@ class CreateContext: def reset_preparation(self): """Prepare attributes that must be prepared/cleaned before reset.""" - pass # Give ability to store shared data for collection phase self._collection_shared_data = {} def reset_finalization(self): """Cleanup of attributes after reset.""" - pass # Stop access to collection shared data self._collection_shared_data = None @@ -1280,58 +1278,19 @@ class CreateContext: plugins.append(plugin) return plugins - def _validate_collection_shared_data(self): + @property + def collection_shared_data(self): + """Access to shared data that can be used during creator's collection. + + Retruns: + Dict[str, Any]: Shared data. + + Raises: + UnavailableSharedData: When called out of collection phase. + """ + if self._collection_shared_data is None: raise UnavailableSharedData( "Accessed Collection shared data out of collection phase" ) - - def collection_shared_data_contains(self, key): - """Check if collection shared data are set. - - Args: - key (str): Key under which are shared data stored. - - Retruns: - bool: Key is already set. - - Raises: - UnavailableSharedData: When called out of collection phase. - """ - - self._validate_collection_shared_data() - return key in self._collection_shared_data - - def get_collection_shared_data(self, key, default=None): - """Receive shared data during collection phase. - - Args: - key (str): Key under which are shared data stored. - default (Any): Default value if key is not set. - - Returns: - Any: Value stored under the key. - - Raises: - UnavailableSharedData: When called out of collection phase. - """ - - self._validate_collection_shared_data() - return self._collection_shared_data.get(key, default) - - def set_collection_shared_data(self, key, value): - """Store a value under collection shared data. - - It is highly recommended to use very specific keys as creators may - clash each other if simple keys are used. - - Args: - key (str): Key under which will shared data be stored. - value (Any): Value to store. - - Raises: - UnavailableSharedData: When called out of collection phase. - """ - - self._validate_collection_shared_data() - self._collection_shared_data[key] = value + return self._collection_shared_data diff --git a/openpype/pipeline/create/creator_plugins.py b/openpype/pipeline/create/creator_plugins.py index e5018c395e..97ee94c449 100644 --- a/openpype/pipeline/create/creator_plugins.py +++ b/openpype/pipeline/create/creator_plugins.py @@ -324,43 +324,18 @@ class BaseCreator: return self.instance_attr_defs - def collection_shared_data_contains(self, key): - """Check if collection shared data are set. - - Args: - key (str): Key under which are shared data stored. + @property + def collection_shared_data(self): + """Access to shared data that can be used during creator's collection. Retruns: - bool: Key is already set. + Dict[str, Any]: Shared data. + + Raises: + UnavailableSharedData: When called out of collection phase. """ - return self.create_context.collection_shared_data_contains(key) - - def get_collection_shared_data(self, key, default=None): - """Receive shared data during collection phase. - - Args: - key (str): Key under which are shared data stored. - default (Any): Default value if key is not set. - - Returns: - Any: Value stored under the key. - """ - - return self.create_context.get_collection_shared_data(key, default) - - def set_collection_shared_data(self, key, value): - """Store a value under collection shared data. - - It is highly recommended to use very specific keys as creators may - clash each other if simple keys are used. - - Args: - key (str): Key under which will shared data be stored. - value (Any): Value to store. - """ - - return self.create_context.set_collection_shared_data(key, value) + return self.create_context.collection_shared_data class Creator(BaseCreator): From 7571d62709eb5c9ecd08a07062e818f32eb4ab0c Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Tue, 18 Oct 2022 15:22:50 +0200 Subject: [PATCH 09/10] Updated documentation --- website/docs/dev_publishing.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/website/docs/dev_publishing.md b/website/docs/dev_publishing.md index 5f30f7f9c8..135f6cd985 100644 --- a/website/docs/dev_publishing.md +++ b/website/docs/dev_publishing.md @@ -73,10 +73,7 @@ Main responsibility of create plugin is to create, update, collect and remove in Base implementation of creator plugin. It is not recommended to use this class as base for production plugins but rather use one of **HiddenCreator**, **AutoCreator** and **Creator** variants. **Access to shared data** -Functions to work with "Collection shared data" can be used during reset phase of `CreateContext`. Creators can cache there data that are common for them. For example list of nodes in scene. Methods are implemented on `CreateContext` but their usage is primarily for Create plugins as nothing else should use it. -- **`collection_shared_data_contains`** - Check if shared data already has set a key. -- **`get_collection_shared_data`** - Receive value of shared data by a key. -- **`set_collection_shared_data`** - Set or update value of shared data key. +Functions to work with "Collection shared data" can be used during reset phase of `CreateContext`. Creators can cache there data that are common for them. For example list of nodes in scene. Methods are implemented on `CreateContext` but their usage is primarily for Create plugins as nothing else should use it. Each creator can access `collection_shared_data` attribute which is a dictionary where shared data can be stored. **Abstractions** - **`family`** (class attr) - Tells what kind of instance will be created. From 0c899e601b22c6c5c9a8f084d4a7a9ad71b8104c Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 20 Oct 2022 17:10:09 +0200 Subject: [PATCH 10/10] fix attributes --- openpype/tools/publisher/control.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/tools/publisher/control.py b/openpype/tools/publisher/control.py index bf1564597f..d2d01e7921 100644 --- a/openpype/tools/publisher/control.py +++ b/openpype/tools/publisher/control.py @@ -1669,7 +1669,7 @@ class PublisherController(BasePublisherController): self.host_is_valid = self._create_context.host_is_valid - self.create_context.reset_preparation() + self._create_context.reset_preparation() # Reset avalon context self._create_context.reset_avalon_context() @@ -1681,7 +1681,7 @@ class PublisherController(BasePublisherController): self._reset_publish() self._reset_instances() - self.create_context.reset_finalization() + self._create_context.reset_finalization() self._emit_event("controller.reset.finished")