From 2d601d051a9b59509c6af159c06f8424591af444 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 29 Jul 2022 16:42:54 +0200 Subject: [PATCH 01/28] give ability to query by representation context and regex --- openpype/client/entities.py | 108 +++++++++++++++++++++++++++++++----- 1 file changed, 94 insertions(+), 14 deletions(-) diff --git a/openpype/client/entities.py b/openpype/client/entities.py index dd5d831ecf..57c38784b0 100644 --- a/openpype/client/entities.py +++ b/openpype/client/entities.py @@ -7,6 +7,7 @@ that has project name as a context (e.g. on 'ProjectEntity'?). """ import os +import re import collections import six @@ -1035,17 +1036,70 @@ def get_representation_by_name( return conn.find_one(query_filter, _prepare_fields(fields)) +def _flatten_dict(data): + flatten_queue = collections.deque() + flatten_queue.append(data) + output = {} + while flatten_queue: + item = flatten_queue.popleft() + for key, value in item.items(): + if not isinstance(value, dict): + output[key] = value + continue + + tmp = {} + for subkey, subvalue in value.items(): + new_key = "{}.{}".format(key, subkey) + tmp[new_key] = subvalue + flatten_queue.append(tmp) + return output + + +def _regex_filters(filters): + output = [] + for key, value in filters.items(): + regexes = [] + a_values = [] + if isinstance(value, re.Pattern): + regexes.append(value) + elif isinstance(value, (list, tuple, set)): + for item in value: + if isinstance(item, re.Pattern): + regexes.append(item) + else: + a_values.append(item) + else: + a_values.append(value) + + key_filters = [] + if len(a_values) == 1: + key_filters.append({key: a_values[0]}) + elif a_values: + key_filters.append({key: {"$in": a_values}}) + + for regex in regexes: + key_filters.append({key: {"$regex": regex}}) + + if len(key_filters) == 1: + output.append(key_filters[0]) + else: + output.append({"$or": key_filters}) + + return output + + def _get_representations( project_name, representation_ids, representation_names, version_ids, - extensions, + context_filters, names_by_version_ids, standard, archived, fields ): + default_output = [] repre_types = [] if standard: repre_types.append("representation") @@ -1053,7 +1107,7 @@ def _get_representations( repre_types.append("archived_representation") if not repre_types: - return [] + return default_output if len(repre_types) == 1: query_filter = {"type": repre_types[0]} @@ -1063,25 +1117,21 @@ def _get_representations( if representation_ids is not None: representation_ids = _convert_ids(representation_ids) if not representation_ids: - return [] + return default_output query_filter["_id"] = {"$in": representation_ids} if representation_names is not None: if not representation_names: - return [] + return default_output query_filter["name"] = {"$in": list(representation_names)} if version_ids is not None: version_ids = _convert_ids(version_ids) if not version_ids: - return [] + return default_output query_filter["parent"] = {"$in": version_ids} - if extensions is not None: - if not extensions: - return [] - query_filter["context.ext"] = {"$in": list(extensions)} - + or_queries = [] if names_by_version_ids is not None: or_query = [] for version_id, names in names_by_version_ids.items(): @@ -1091,8 +1141,35 @@ def _get_representations( "name": {"$in": list(names)} }) if not or_query: + return default_output + or_queries.append(or_query) + + if context_filters is not None: + if not context_filters: return [] - query_filter["$or"] = or_query + _flatten_filters = _flatten_dict(context_filters) + flatten_filters = {} + for key, value in _flatten_filters.items(): + if not key.startswith("context"): + key = "context.{}".format(key) + flatten_filters[key] = value + + for item in _regex_filters(flatten_filters): + for key, value in item.items(): + if key == "$or": + or_queries.append(value) + else: + query_filter[key] = value + + if len(or_queries) == 1: + query_filter["$or"] = or_queries[0] + elif or_queries: + and_query = [] + for or_query in or_queries: + if isinstance(or_query, list): + or_query = {"$or": or_query} + and_query.append(or_query) + query_filter["$and"] = and_query conn = get_project_connection(project_name) @@ -1104,7 +1181,7 @@ def get_representations( representation_ids=None, representation_names=None, version_ids=None, - extensions=None, + context_filters=None, names_by_version_ids=None, archived=False, standard=True, @@ -1122,8 +1199,8 @@ def get_representations( as filter. Filter ignored if 'None' is passed. version_ids (Iterable[str]): Subset ids used as parent filter. Filter ignored if 'None' is passed. - extensions (Iterable[str]): Filter by extension of main representation - file (without dot). + context_filters (Dict[str, List[str, re.Pattern]]): Filter by + representation context fields. names_by_version_ids (dict[ObjectId, list[str]]): Complex filtering using version ids and list of names under the version. archived (bool): Output will also contain archived representations. @@ -1140,6 +1217,7 @@ def get_representations( representation_names=representation_names, version_ids=version_ids, extensions=extensions, + context_filters=context_filters, names_by_version_ids=names_by_version_ids, standard=True, archived=archived, @@ -1153,6 +1231,7 @@ def get_archived_representations( representation_names=None, version_ids=None, extensions=None, + context_filters=None, names_by_version_ids=None, fields=None ): @@ -1185,6 +1264,7 @@ def get_archived_representations( representation_names=representation_names, version_ids=version_ids, extensions=extensions, + context_filters=context_filters, names_by_version_ids=names_by_version_ids, standard=False, archived=True, From c65dd9747f5197868a9153fc109915ed654122ab Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 29 Jul 2022 17:15:13 +0200 Subject: [PATCH 02/28] added new method 'get_representations' to get representations from placeholder --- .../workfile/abstract_template_loader.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/openpype/pipeline/workfile/abstract_template_loader.py b/openpype/pipeline/workfile/abstract_template_loader.py index 00bc8f15a7..0a422f5cca 100644 --- a/openpype/pipeline/workfile/abstract_template_loader.py +++ b/openpype/pipeline/workfile/abstract_template_loader.py @@ -456,8 +456,25 @@ class AbstractPlaceholder: @abstractmethod def clean(self): - """Clean placeholder from hierarchy after loading assets. + """Clean placeholder from hierarchy after loading assets.""" + + pass + + @abstractmethod + def get_representations(self, current_asset, linked_assets): + """Query representations based on placeholder data. + + Args: + current_asset (str): Name of current + context asset. + linked_assets (List[str]): Names of assets + linked to current context asset. + + Returns: + Iterable[Dict[str, Any]]: Representations that are matching + placeholder filters. """ + pass @abstractmethod From da8e25f4a1b7ea89bf9c7cac62c8a3ea10fbb9e6 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 29 Jul 2022 17:16:23 +0200 Subject: [PATCH 03/28] use 'get_representations' instead of 'convert_to_db_filters' --- .../workfile/abstract_template_loader.py | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/openpype/pipeline/workfile/abstract_template_loader.py b/openpype/pipeline/workfile/abstract_template_loader.py index 0a422f5cca..a2505c061e 100644 --- a/openpype/pipeline/workfile/abstract_template_loader.py +++ b/openpype/pipeline/workfile/abstract_template_loader.py @@ -280,19 +280,16 @@ class AbstractTemplateLoader: def get_placeholder_representations( self, placeholder, current_asset, linked_assets ): - # TODO This approach must be changed. Placeholders should return - # already prepared data and not query them here. - # - this is impossible to handle using query functions - placeholder_db_filters = placeholder.convert_to_db_filters( + placeholder_representations = placeholder.get_representations( current_asset, - linked_assets) - # get representation by assets - for db_filter in placeholder_db_filters: - placeholder_representations = list(legacy_io.find(db_filter)) - for representation in reduce(update_representations, - placeholder_representations, - dict()).values(): - yield representation + linked_assets + ) + for repre_doc in reduce( + update_representations, + placeholder_representations, + dict() + ).values(): + yield repre_doc def load_data_is_incorrect( self, placeholder, last_representation, ignored_ids): From 0e0cec5e0146a3001a4a349360324346fd0ab961 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 29 Jul 2022 17:19:47 +0200 Subject: [PATCH 04/28] pass asset documents instead of just names --- .../workfile/abstract_template_loader.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/openpype/pipeline/workfile/abstract_template_loader.py b/openpype/pipeline/workfile/abstract_template_loader.py index a2505c061e..96012eba36 100644 --- a/openpype/pipeline/workfile/abstract_template_loader.py +++ b/openpype/pipeline/workfile/abstract_template_loader.py @@ -223,10 +223,10 @@ class AbstractTemplateLoader: Returns: None """ + loaders_by_name = self.loaders_by_name - current_asset = self.current_asset - linked_assets = [asset['name'] for asset - in get_linked_assets(self.current_asset_doc)] + current_asset_doc = self.current_asset_doc + linked_assets = get_linked_assets(current_asset_doc) ignored_ids = ignored_ids or [] placeholders = self.get_placeholders() @@ -239,7 +239,7 @@ class AbstractTemplateLoader: )) placeholder_representations = self.get_placeholder_representations( placeholder, - current_asset, + current_asset_doc, linked_assets ) @@ -278,11 +278,11 @@ class AbstractTemplateLoader: self.postload(placeholder) def get_placeholder_representations( - self, placeholder, current_asset, linked_assets + self, placeholder, current_asset_doc, linked_asset_docs ): placeholder_representations = placeholder.get_representations( - current_asset, - linked_assets + current_asset_doc, + linked_asset_docs ) for repre_doc in reduce( update_representations, @@ -458,13 +458,13 @@ class AbstractPlaceholder: pass @abstractmethod - def get_representations(self, current_asset, linked_assets): + def get_representations(self, current_asset_doc, linked_asset_docs): """Query representations based on placeholder data. Args: - current_asset (str): Name of current + current_asset_doc (Dict[str, Any]): Document of current context asset. - linked_assets (List[str]): Names of assets + linked_asset_docs (List[Dict[str, Any]]): Documents of assets linked to current context asset. Returns: From ef674857f85f360954b4d6e2c6f6c0c4acf3f711 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 29 Jul 2022 17:29:34 +0200 Subject: [PATCH 05/28] implemented get_representations for maya placeholder --- openpype/hosts/maya/api/template_loader.py | 80 +++++++++++----------- 1 file changed, 41 insertions(+), 39 deletions(-) diff --git a/openpype/hosts/maya/api/template_loader.py b/openpype/hosts/maya/api/template_loader.py index 6b225442e7..f553730186 100644 --- a/openpype/hosts/maya/api/template_loader.py +++ b/openpype/hosts/maya/api/template_loader.py @@ -1,5 +1,7 @@ +import re from maya import cmds +from openpype.client import get_representations from openpype.pipeline import legacy_io from openpype.pipeline.workfile.abstract_template_loader import ( AbstractPlaceholder, @@ -191,48 +193,48 @@ class MayaPlaceholder(AbstractPlaceholder): cmds.hide(node) cmds.setAttr(node + '.hiddenInOutliner', True) - def convert_to_db_filters(self, current_asset, linked_asset): - if self.data['builder_type'] == "context_asset": - return [ - { - "type": "representation", - "context.asset": { - "$eq": current_asset, - "$regex": self.data['asset'] - }, - "context.subset": {"$regex": self.data['subset']}, - "context.hierarchy": {"$regex": self.data['hierarchy']}, - "context.representation": self.data['representation'], - "context.family": self.data['family'], - } - ] + def get_representations(self, current_asset_doc, linked_asset_docs): + project_name = legacy_io.active_project() - elif self.data['builder_type'] == "linked_asset": - return [ - { - "type": "representation", - "context.asset": { - "$eq": asset_name, - "$regex": self.data['asset'] - }, - "context.subset": {"$regex": self.data['subset']}, - "context.hierarchy": {"$regex": self.data['hierarchy']}, - "context.representation": self.data['representation'], - "context.family": self.data['family'], - } for asset_name in linked_asset - ] + builder_type = self.data["builder_type"] + if builder_type == "context_asset": + context_filters = { + "asset": [current_asset_doc["name"]], + "subset": [re.compile(self.data["subset"])], + "hierarchy": [re.compile(self.data["hierarchy"])], + "representations": [self.data["representation"]], + "family": [self.data["family"]] + } + + elif builder_type != "linked_asset": + context_filters = { + "asset": [re.compile(self.data["asset"])], + "subset": [re.compile(self.data["subset"])], + "hierarchy": [re.compile(self.data["hierarchy"])], + "representation": [self.data["representation"]], + "family": [self.data["family"]] + } else: - return [ - { - "type": "representation", - "context.asset": {"$regex": self.data['asset']}, - "context.subset": {"$regex": self.data['subset']}, - "context.hierarchy": {"$regex": self.data['hierarchy']}, - "context.representation": self.data['representation'], - "context.family": self.data['family'], - } - ] + asset_regex = re.compile(self.data["asset"]) + linked_asset_names = [] + for asset_doc in linked_asset_docs: + asset_name = asset_doc["name"] + if asset_regex.match(asset_name): + linked_asset_names.append(asset_name) + + context_filters = { + "asset": linked_asset_names, + "subset": [re.compile(self.data["subset"])], + "hierarchy": [re.compile(self.data["hierarchy"])], + "representation": [self.data["representation"]], + "family": [self.data["family"]], + } + + return list(get_representations( + project_name, + context_filters=context_filters + )) def err_message(self): return ( From a6406f72d36d8eb748404af8fc6e6d61c6c6b451 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 29 Jul 2022 17:39:57 +0200 Subject: [PATCH 06/28] added logger to placeholder --- .../pipeline/workfile/abstract_template_loader.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/openpype/pipeline/workfile/abstract_template_loader.py b/openpype/pipeline/workfile/abstract_template_loader.py index 96012eba36..d934c50daf 100644 --- a/openpype/pipeline/workfile/abstract_template_loader.py +++ b/openpype/pipeline/workfile/abstract_template_loader.py @@ -397,8 +397,19 @@ class AbstractPlaceholder: optional_attributes = {} def __init__(self, node): + self._log = None self.get_data(node) + @property + def log(self): + if self._log is None: + self._log = Logger.get_logger(repr(self)) + return self._log + + def __repr__(self): + return "< {} {} >".format(self.__class__.__name__, self.name) + + def order(self): """Get placeholder order. Order is used to sort them by priority @@ -436,9 +447,9 @@ class AbstractPlaceholder: Bool: True if every attributes are a key of data """ if set(self.attributes).issubset(self.data.keys()): - print("Valid placeholder : {}".format(self.data["node"])) + self.log.debug("Valid placeholder: {}".format(self.data["node"])) return True - print("Placeholder is not valid : {}".format(self.data["node"])) + self.log.info("Placeholder is not valid: {}".format(self.data["node"])) return False @abstractmethod From 8b7531b97775d3facefde41682dd19e9dd3e11f6 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 29 Jul 2022 17:44:03 +0200 Subject: [PATCH 07/28] added helper attributes to placeholder so there is no need to access it's 'data' --- .../workfile/abstract_template_loader.py | 50 +++++++++++++------ 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/openpype/pipeline/workfile/abstract_template_loader.py b/openpype/pipeline/workfile/abstract_template_loader.py index d934c50daf..5ecc154ea4 100644 --- a/openpype/pipeline/workfile/abstract_template_loader.py +++ b/openpype/pipeline/workfile/abstract_template_loader.py @@ -231,11 +231,11 @@ class AbstractTemplateLoader: ignored_ids = ignored_ids or [] placeholders = self.get_placeholders() self.log.debug("Placeholders found in template: {}".format( - [placeholder.data['node'] for placeholder in placeholders] + [placeholder.name] for placeholder in placeholders] )) for placeholder in placeholders: self.log.debug("Start to processing placeholder {}".format( - placeholder.data['node'] + placeholder.name )) placeholder_representations = self.get_placeholder_representations( placeholder, @@ -246,7 +246,7 @@ class AbstractTemplateLoader: if not placeholder_representations: self.log.info( "There's no representation for this placeholder: " - "{}".format(placeholder.data['node']) + "{}".format(placeholder.name) ) continue @@ -264,8 +264,8 @@ class AbstractTemplateLoader: "Loader arguments used : {}".format( representation['context']['asset'], representation['context']['subset'], - placeholder.loader, - placeholder.data['loader_args'])) + placeholder.loader_name, + placeholder.loader_args)) try: container = self.load( @@ -307,19 +307,22 @@ class AbstractTemplateLoader: def load(self, placeholder, loaders_by_name, last_representation): repre = get_representation_context(last_representation) return load_with_repre_context( - loaders_by_name[placeholder.loader], + loaders_by_name[placeholder.loader_name], repre, - options=parse_loader_args(placeholder.data['loader_args'])) + options=parse_loader_args(placeholder.loader_args)) def load_succeed(self, placeholder, container): placeholder.parent_in_hierarchy(container) def load_failed(self, placeholder, last_representation): - self.log.warning("Got error trying to load {}:{} with {}\n\n" - "{}".format(last_representation['context']['asset'], - last_representation['context']['subset'], - placeholder.loader, - traceback.format_exc())) + self.log.warning( + "Got error trying to load {}:{} with {}".format( + last_representation['context']['asset'], + last_representation['context']['subset'], + placeholder.loader_name + ), + exc_info=True + ) def postload(self, placeholder): placeholder.clean() @@ -398,6 +401,7 @@ class AbstractPlaceholder: def __init__(self, node): self._log = None + self._name = node self.get_data(node) @property @@ -409,6 +413,17 @@ class AbstractPlaceholder: def __repr__(self): return "< {} {} >".format(self.__class__.__name__, self.name) + @property + def name(self): + return self._name + + @property + def loader_args(self): + return self.data["loader_args"] + + @property + def builder_type(self): + return self.data["builder_type"] def order(self): """Get placeholder order. @@ -423,12 +438,15 @@ class AbstractPlaceholder: return self.data.get('order') @property - def loader(self): - """Return placeholder loader type + def loader_name(self): + """Return placeholder loader type. + Returns: - string: Loader name + str: Loader name that will be used to load placeholder + representations. """ - return self.data.get('loader') + + return self.data["loader"] @property def is_context(self): From 2d7910a26410936f1d23282b9011780cccfc8680 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 29 Jul 2022 17:46:14 +0200 Subject: [PATCH 08/28] renamed 'attributes' to 'required_keys' and 'optional_attributes' to 'optional_keys' --- openpype/hosts/maya/api/template_loader.py | 8 +++-- .../workfile/abstract_template_loader.py | 35 ++++++++++++------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/openpype/hosts/maya/api/template_loader.py b/openpype/hosts/maya/api/template_loader.py index f553730186..ecffafc93d 100644 --- a/openpype/hosts/maya/api/template_loader.py +++ b/openpype/hosts/maya/api/template_loader.py @@ -98,11 +98,11 @@ class MayaPlaceholder(AbstractPlaceholder): """Concrete implementation of AbstractPlaceholder for maya """ - optional_attributes = {'asset', 'subset', 'hierarchy'} + optional_keys = {'asset', 'subset', 'hierarchy'} def get_data(self, node): user_data = dict() - for attr in self.attributes.union(self.optional_attributes): + for attr in self.required_keys.union(self.optional_keys): attribute_name = '{}.{}'.format(node, attr) if not cmds.attributeQuery(attr, node=node, exists=True): print("{} not found".format(attribute_name)) @@ -112,7 +112,9 @@ class MayaPlaceholder(AbstractPlaceholder): asString=True) user_data['parent'] = ( cmds.getAttr(node + '.parent', asString=True) - or node.rpartition('|')[0] or "") + or node.rpartition('|')[0] + or "" + ) user_data['node'] = node if user_data['parent']: siblings = cmds.listRelatives(user_data['parent'], children=True) diff --git a/openpype/pipeline/workfile/abstract_template_loader.py b/openpype/pipeline/workfile/abstract_template_loader.py index 5ecc154ea4..56fb31fa0c 100644 --- a/openpype/pipeline/workfile/abstract_template_loader.py +++ b/openpype/pipeline/workfile/abstract_template_loader.py @@ -377,15 +377,17 @@ class AbstractTemplateLoader: @six.add_metaclass(ABCMeta) class AbstractPlaceholder: - """Abstraction of placeholders logic + """Abstraction of placeholders logic. + Properties: - attributes: A list of mandatory attribute to decribe placeholder + required_keys: A list of mandatory keys to decribe placeholder and assets to load. - optional_attributes: A list of optional attribute to decribe + optional_keys: A list of optional keys to decribe placeholder and assets to load loader: Name of linked loader to use while loading assets is_context: Is placeholder linked to context asset (or to linked assets) + Methods: is_repres_valid: loader: @@ -395,9 +397,15 @@ class AbstractPlaceholder: parent_in_hierachy: """ - attributes = {'builder_type', 'family', 'representation', - 'order', 'loader', 'loader_args'} - optional_attributes = {} + required_keys = { + "builder_type", + "family", + "representation", + "order", + "loader", + "loader_args" + } + optional_keys = {} def __init__(self, node): self._log = None @@ -459,15 +467,18 @@ class AbstractPlaceholder: return self.data.get('builder_type') == 'context_asset' def is_valid(self): - """Test validity of placeholder - i.e.: every attributes exists in placeholder data + """Test validity of placeholder. + + i.e.: every required key exists in placeholder data + Returns: - Bool: True if every attributes are a key of data + bool: True if every key is in data """ - if set(self.attributes).issubset(self.data.keys()): - self.log.debug("Valid placeholder: {}".format(self.data["node"])) + + if set(self.required_keys).issubset(self.data.keys()): + self.log.debug("Valid placeholder : {}".format(self.name)) return True - self.log.info("Placeholder is not valid: {}".format(self.data["node"])) + self.log.info("Placeholder is not valid : {}".format(self.name)) return False @abstractmethod From 736123d1c2496df1604d1b0c84df5a2646cc51f9 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 29 Jul 2022 17:48:47 +0200 Subject: [PATCH 09/28] modified 'is_context' property --- .../pipeline/workfile/abstract_template_loader.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/openpype/pipeline/workfile/abstract_template_loader.py b/openpype/pipeline/workfile/abstract_template_loader.py index 56fb31fa0c..a1d188ea6c 100644 --- a/openpype/pipeline/workfile/abstract_template_loader.py +++ b/openpype/pipeline/workfile/abstract_template_loader.py @@ -458,14 +458,22 @@ class AbstractPlaceholder: @property def is_context(self): - """Return placeholder type + """Check if is placeholder context type. + context_asset: For loading current asset linked_asset: For loading linked assets + + Question: + There seems to be more build options and this property is not used, + should be removed? + Returns: bool: true if placeholder is a context placeholder """ - return self.data.get('builder_type') == 'context_asset' + return self.builder_type == "context_asset" + + @property def is_valid(self): """Test validity of placeholder. From 49799c2d8871a52fb1fd8210b31a1e51fd5f3f2b Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 4 Aug 2022 18:26:38 +0200 Subject: [PATCH 10/28] fix merge conflict --- openpype/pipeline/workfile/abstract_template_loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/pipeline/workfile/abstract_template_loader.py b/openpype/pipeline/workfile/abstract_template_loader.py index a1d188ea6c..5d8d79397a 100644 --- a/openpype/pipeline/workfile/abstract_template_loader.py +++ b/openpype/pipeline/workfile/abstract_template_loader.py @@ -231,7 +231,7 @@ class AbstractTemplateLoader: ignored_ids = ignored_ids or [] placeholders = self.get_placeholders() self.log.debug("Placeholders found in template: {}".format( - [placeholder.name] for placeholder in placeholders] + [placeholder.name for placeholder in placeholders] )) for placeholder in placeholders: self.log.debug("Start to processing placeholder {}".format( From 7d1f1bb064190873beee61c0a4eb4df598747c88 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 5 Aug 2022 09:50:11 +0200 Subject: [PATCH 11/28] remove extensions arguments --- openpype/client/entities.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/openpype/client/entities.py b/openpype/client/entities.py index 57c38784b0..a3fcd01f80 100644 --- a/openpype/client/entities.py +++ b/openpype/client/entities.py @@ -1216,7 +1216,6 @@ def get_representations( representation_ids=representation_ids, representation_names=representation_names, version_ids=version_ids, - extensions=extensions, context_filters=context_filters, names_by_version_ids=names_by_version_ids, standard=True, @@ -1230,7 +1229,6 @@ def get_archived_representations( representation_ids=None, representation_names=None, version_ids=None, - extensions=None, context_filters=None, names_by_version_ids=None, fields=None @@ -1247,8 +1245,6 @@ def get_archived_representations( as filter. Filter ignored if 'None' is passed. version_ids (Iterable[str]): Subset ids used as parent filter. Filter ignored if 'None' is passed. - extensions (Iterable[str]): Filter by extension of main representation - file (without dot). names_by_version_ids (dict[ObjectId, List[str]]): Complex filtering using version ids and list of names under the version. fields (Iterable[str]): Fields that should be returned. All fields are @@ -1263,7 +1259,6 @@ def get_archived_representations( representation_ids=representation_ids, representation_names=representation_names, version_ids=version_ids, - extensions=extensions, context_filters=context_filters, names_by_version_ids=names_by_version_ids, standard=False, From 8db8ada9642bcdf2c5f364fbf78c902344b1613e Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 8 Aug 2022 17:46:32 +0200 Subject: [PATCH 12/28] changed 'node' variable to 'identifier' and added it's docstrings --- .../workfile/abstract_template_loader.py | 20 +++++++------------ 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/openpype/pipeline/workfile/abstract_template_loader.py b/openpype/pipeline/workfile/abstract_template_loader.py index 5d8d79397a..16287bbd4e 100644 --- a/openpype/pipeline/workfile/abstract_template_loader.py +++ b/openpype/pipeline/workfile/abstract_template_loader.py @@ -384,17 +384,11 @@ class AbstractPlaceholder: and assets to load. optional_keys: A list of optional keys to decribe placeholder and assets to load - loader: Name of linked loader to use while loading assets - is_context: Is placeholder linked - to context asset (or to linked assets) + loader_name: Name of linked loader to use while loading assets - Methods: - is_repres_valid: - loader: - order: - is_valid: - get_data: - parent_in_hierachy: + Args: + identifier (str): Placeholder identifier. Should be possible to be + used as identifier in "a scene" (e.g. unique node name). """ required_keys = { @@ -407,10 +401,10 @@ class AbstractPlaceholder: } optional_keys = {} - def __init__(self, node): + def __init__(self, identifier): self._log = None - self._name = node - self.get_data(node) + self._name = identifier + self.get_data(identifier) @property def log(self): From 5d0cd42a8133bcf7d65bbcef0c7b093ef058d7b2 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 8 Aug 2022 17:47:01 +0200 Subject: [PATCH 13/28] renamed 'order' method to 'get_order' --- .../pipeline/workfile/abstract_template_loader.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/openpype/pipeline/workfile/abstract_template_loader.py b/openpype/pipeline/workfile/abstract_template_loader.py index 16287bbd4e..fe1f15c140 100644 --- a/openpype/pipeline/workfile/abstract_template_loader.py +++ b/openpype/pipeline/workfile/abstract_template_loader.py @@ -336,7 +336,7 @@ class AbstractTemplateLoader: placeholders = map(placeholder_class, self.get_template_nodes()) valid_placeholders = filter(placeholder_class.is_valid, placeholders) sorted_placeholders = sorted(valid_placeholders, - key=placeholder_class.order) + key=placeholder_class.get_order) return sorted_placeholders @abstractmethod @@ -427,17 +427,24 @@ class AbstractPlaceholder: def builder_type(self): return self.data["builder_type"] + @property def order(self): - """Get placeholder order. + return self.data["order"] + + def get_order(self): + """Placeholder order. + Order is used to sort them by priority Priority is lowset first, highest last (ex: 1: First to load 100: Last to load) + Returns: - Int: Order priority + int: Order priority """ - return self.data.get('order') + + return self.order @property def loader_name(self): From 7e8e61c0e4d51334d6de0d1f9cd672fa0dae5313 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 8 Aug 2022 17:48:14 +0200 Subject: [PATCH 14/28] changed 'get_data' docstring --- openpype/pipeline/workfile/abstract_template_loader.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/openpype/pipeline/workfile/abstract_template_loader.py b/openpype/pipeline/workfile/abstract_template_loader.py index fe1f15c140..66943eafe7 100644 --- a/openpype/pipeline/workfile/abstract_template_loader.py +++ b/openpype/pipeline/workfile/abstract_template_loader.py @@ -537,10 +537,12 @@ class AbstractPlaceholder: pass @abstractmethod - def get_data(self, node): - """ - Collect placeholders information. + def get_data(self, identifier): + """Collect information about placeholder by identifier. + Args: - node (AnyNode): A unique node decided by Placeholder implementation + identifier (str): A unique placeholder identifier defined by + implementation. """ + pass From a1cd1890d6db952e4feee357e204444aed0015ed Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 8 Aug 2022 17:48:28 +0200 Subject: [PATCH 15/28] modified 'parent_in_hierarchy' docstring --- openpype/pipeline/workfile/abstract_template_loader.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/openpype/pipeline/workfile/abstract_template_loader.py b/openpype/pipeline/workfile/abstract_template_loader.py index 66943eafe7..a1629d9b79 100644 --- a/openpype/pipeline/workfile/abstract_template_loader.py +++ b/openpype/pipeline/workfile/abstract_template_loader.py @@ -491,13 +491,13 @@ class AbstractPlaceholder: return False @abstractmethod - def parent_in_hierarchy(self, containers): - """Place container in correct hierarchy - given by placeholder + def parent_in_hierarchy(self, container): + """Place loaded container in correct hierarchy given by placeholder + Args: - containers (String): Container name returned back by - placeholder's loader. + container (Dict[str, Any]): Loaded container created by loader. """ + pass @abstractmethod From 56150d4abb72d8b0025a7724e002eab792aa34a0 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 8 Aug 2022 17:48:48 +0200 Subject: [PATCH 16/28] removed unused method 'convert_to_db_filters' --- .../pipeline/workfile/abstract_template_loader.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/openpype/pipeline/workfile/abstract_template_loader.py b/openpype/pipeline/workfile/abstract_template_loader.py index a1629d9b79..c36e489017 100644 --- a/openpype/pipeline/workfile/abstract_template_loader.py +++ b/openpype/pipeline/workfile/abstract_template_loader.py @@ -523,19 +523,6 @@ class AbstractPlaceholder: pass - @abstractmethod - def convert_to_db_filters(self, current_asset, linked_asset): - """map current placeholder data as a db filter - args: - current_asset (String): Name of current asset in context - linked asset (list[String]) : Names of assets linked to - current asset in context - Returns: - dict: a dictionnary describing a filter to look for asset in - a database - """ - pass - @abstractmethod def get_data(self, identifier): """Collect information about placeholder by identifier. From 56bbbdbd583b51ba07bac08e753fb5a2050a768f Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 8 Aug 2022 17:49:20 +0200 Subject: [PATCH 17/28] removed unused import --- openpype/pipeline/workfile/abstract_template_loader.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/openpype/pipeline/workfile/abstract_template_loader.py b/openpype/pipeline/workfile/abstract_template_loader.py index c36e489017..725ab1dab3 100644 --- a/openpype/pipeline/workfile/abstract_template_loader.py +++ b/openpype/pipeline/workfile/abstract_template_loader.py @@ -1,8 +1,6 @@ import os from abc import ABCMeta, abstractmethod -import traceback - import six import logging from functools import reduce From 02edebad41f26680f0f7ceb3b2b21fe6cfebebab Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 10 Aug 2022 18:33:03 +0200 Subject: [PATCH 18/28] fix import string --- openpype/pipeline/workfile/build_template.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/pipeline/workfile/build_template.py b/openpype/pipeline/workfile/build_template.py index df6fe3514a..e6396578c5 100644 --- a/openpype/pipeline/workfile/build_template.py +++ b/openpype/pipeline/workfile/build_template.py @@ -15,7 +15,7 @@ from .build_template_exceptions import ( MissingTemplateLoaderClass ) -_module_path_format = 'openpype.{host}.template_loader' +_module_path_format = 'openpype.hosts.{host}.api.template_loader' def build_workfile_template(*args): From 25616886bff2b6fda0b4c9646ea9256389ba248f Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 11 Aug 2022 15:08:14 +0200 Subject: [PATCH 19/28] raise and error when nothing is selected --- openpype/hosts/maya/api/lib_template_builder.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/openpype/hosts/maya/api/lib_template_builder.py b/openpype/hosts/maya/api/lib_template_builder.py index 855c72e361..34a8450a26 100644 --- a/openpype/hosts/maya/api/lib_template_builder.py +++ b/openpype/hosts/maya/api/lib_template_builder.py @@ -40,6 +40,9 @@ def create_placeholder(): placeholder_name = create_placeholder_name(args, options) selection = cmds.ls(selection=True) + if not selection: + raise ValueError("Nothing is selected") + placeholder = cmds.spaceLocator(name=placeholder_name)[0] # get the long name of the placeholder (with the groups) From 683468c5633a42b8c5e80510ab060f981452d02c Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 11 Aug 2022 15:22:08 +0200 Subject: [PATCH 20/28] use 'filter_profiles' function for profiles filtering --- .../workfile/abstract_template_loader.py | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/openpype/pipeline/workfile/abstract_template_loader.py b/openpype/pipeline/workfile/abstract_template_loader.py index 725ab1dab3..51d06cdb3f 100644 --- a/openpype/pipeline/workfile/abstract_template_loader.py +++ b/openpype/pipeline/workfile/abstract_template_loader.py @@ -7,7 +7,11 @@ from functools import reduce from openpype.client import get_asset_by_name from openpype.settings import get_project_settings -from openpype.lib import get_linked_assets, PypeLogger as Logger +from openpype.lib import ( + Logger, + filter_profiles, + get_linked_assets, +) from openpype.pipeline import legacy_io, Anatomy from openpype.pipeline.load import ( get_loaders_by_name, @@ -167,22 +171,23 @@ class AbstractTemplateLoader: anatomy = Anatomy(project_name) project_settings = get_project_settings(project_name) - build_info = project_settings[host_name]['templated_workfile_build'] - profiles = build_info['profiles'] + build_info = project_settings[host_name]["templated_workfile_build"] + profile = filter_profiles( + build_info["profiles"], + { + "task_types": task_type, + "tasks": task_name + } + ) - for prf in profiles: - if prf['task_types'] and task_type not in prf['task_types']: - continue - if prf['tasks'] and task_name not in prf['tasks']: - continue - path = prf['path'] - break - else: # IF no template were found (no break happened) + if not profile: raise TemplateProfileNotFound( "No matching profile found for task '{}' of type '{}' " "with host '{}'".format(task_name, task_type, host_name) ) - if path is None: + + path = profile["path"] + if not path: raise TemplateLoadingFailed( "Template path is not set.\n" "Path need to be set in {}\\Template Workfile Build " From bb9a16100acd9a7d94ec6ff6ea15891916eea580 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 11 Aug 2022 15:22:23 +0200 Subject: [PATCH 21/28] removed unnecessary finally statement --- openpype/pipeline/workfile/abstract_template_loader.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/openpype/pipeline/workfile/abstract_template_loader.py b/openpype/pipeline/workfile/abstract_template_loader.py index 51d06cdb3f..0ed32033af 100644 --- a/openpype/pipeline/workfile/abstract_template_loader.py +++ b/openpype/pipeline/workfile/abstract_template_loader.py @@ -205,9 +205,8 @@ class AbstractTemplateLoader: raise KeyError( "Could not solve key '{}' in template path '{}'".format( missing_key, path)) - finally: - solved_path = os.path.normpath(solved_path) + solved_path = os.path.normpath(solved_path) if not os.path.exists(solved_path): raise TemplateNotFound( "Template found in openPype settings for task '{}' with host " From 12a8307a8331334ee9700efba2127211ea332ff0 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 11 Aug 2022 15:40:02 +0200 Subject: [PATCH 22/28] simplified path formatting --- .../workfile/abstract_template_loader.py | 43 +++++++++++++------ 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/openpype/pipeline/workfile/abstract_template_loader.py b/openpype/pipeline/workfile/abstract_template_loader.py index 0ed32033af..5afec56d71 100644 --- a/openpype/pipeline/workfile/abstract_template_loader.py +++ b/openpype/pipeline/workfile/abstract_template_loader.py @@ -8,6 +8,7 @@ from functools import reduce from openpype.client import get_asset_by_name from openpype.settings import get_project_settings from openpype.lib import ( + StringTemplate, Logger, filter_profiles, get_linked_assets, @@ -192,19 +193,35 @@ class AbstractTemplateLoader: "Template path is not set.\n" "Path need to be set in {}\\Template Workfile Build " "Settings\\Profiles".format(host_name.title())) - try: - solved_path = None - while True: + + # Try fill path with environments and anatomy roots + fill_data = { + key: value + for key, value in os.environ.items() + } + fill_data["root"] = anatomy.roots + result = StringTemplate.format_template(path, fill_data) + if result.solved: + path = result.normalized() + + if path and os.path.exists(path): + self.log.info("Found template at: '{}'".format(path)) + return path + + solved_path = None + while True: + try: solved_path = anatomy.path_remapper(path) - if solved_path is None: - solved_path = path - if solved_path == path: - break - path = solved_path - except KeyError as missing_key: - raise KeyError( - "Could not solve key '{}' in template path '{}'".format( - missing_key, path)) + except KeyError as missing_key: + raise KeyError( + "Could not solve key '{}' in template path '{}'".format( + missing_key, path)) + + if solved_path is None: + solved_path = path + if solved_path == path: + break + path = solved_path solved_path = os.path.normpath(solved_path) if not os.path.exists(solved_path): @@ -213,7 +230,7 @@ class AbstractTemplateLoader: "'{}' does not exists. (Not found : {})".format( task_name, host_name, solved_path)) - self.log.info("Found template at : '{}'".format(solved_path)) + self.log.info("Found template at: '{}'".format(solved_path)) return solved_path From 748dcf1ad207edd3dbf3bc98120d9e46bf9b39e0 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 11 Aug 2022 15:51:27 +0200 Subject: [PATCH 23/28] fix filter and sort --- .../workfile/abstract_template_loader.py | 29 ++++++------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/openpype/pipeline/workfile/abstract_template_loader.py b/openpype/pipeline/workfile/abstract_template_loader.py index 5afec56d71..1c8ede25e6 100644 --- a/openpype/pipeline/workfile/abstract_template_loader.py +++ b/openpype/pipeline/workfile/abstract_template_loader.py @@ -351,11 +351,15 @@ class AbstractTemplateLoader: self.populate_template(ignored_ids=loaded_containers_ids) def get_placeholders(self): - placeholder_class = self.placeholder_class - placeholders = map(placeholder_class, self.get_template_nodes()) - valid_placeholders = filter(placeholder_class.is_valid, placeholders) - sorted_placeholders = sorted(valid_placeholders, - key=placeholder_class.get_order) + placeholders = map(self.placeholder_class, self.get_template_nodes()) + valid_placeholders = filter( + lambda i: i.is_valid, + placeholders + ) + sorted_placeholders = list(sorted( + valid_placeholders, + key=lambda i: i.order + )) return sorted_placeholders @abstractmethod @@ -450,21 +454,6 @@ class AbstractPlaceholder: def order(self): return self.data["order"] - def get_order(self): - """Placeholder order. - - Order is used to sort them by priority - Priority is lowset first, highest last - (ex: - 1: First to load - 100: Last to load) - - Returns: - int: Order priority - """ - - return self.order - @property def loader_name(self): """Return placeholder loader type. From 7eaa278c741ceaa30daf056dd17ec9e4b4ceed10 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 11 Aug 2022 15:54:19 +0200 Subject: [PATCH 24/28] removed invalid default setting for templates --- openpype/settings/defaults/project_settings/maya.json | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/openpype/settings/defaults/project_settings/maya.json b/openpype/settings/defaults/project_settings/maya.json index 9c2c737ece..e9109abd22 100644 --- a/openpype/settings/defaults/project_settings/maya.json +++ b/openpype/settings/defaults/project_settings/maya.json @@ -123,7 +123,6 @@ "defaults": [ "Main" ] - }, "CreateAss": { "enabled": true, @@ -969,13 +968,7 @@ ] }, "templated_workfile_build": { - "profiles": [ - { - "task_types": [], - "tasks": [], - "path": "/path/to/your/template" - } - ] + "profiles": [] }, "filters": { "preset 1": { From 66ee0beaf6d0e09eae6a8a9887a90651618a73f2 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 11 Aug 2022 17:43:47 +0200 Subject: [PATCH 25/28] fix empty or query --- openpype/client/entities.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/openpype/client/entities.py b/openpype/client/entities.py index f9d3badb1a..c798c0ad6d 100644 --- a/openpype/client/entities.py +++ b/openpype/client/entities.py @@ -1130,11 +1130,12 @@ def _get_representations( for item in _regex_filters(flatten_filters): for key, value in item.items(): - if key == "$or": - or_queries.append(value) - else: + if key != "$or": query_filter[key] = value + elif value: + or_queries.append(value) + if len(or_queries) == 1: query_filter["$or"] = or_queries[0] elif or_queries: From 089cd3f9fa3587178c9fe73371b4470588b8467b Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 12 Aug 2022 13:55:00 +0200 Subject: [PATCH 26/28] added missing docstring for 'context_filters' argument --- openpype/client/entities.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openpype/client/entities.py b/openpype/client/entities.py index c798c0ad6d..67ddb09ddb 100644 --- a/openpype/client/entities.py +++ b/openpype/client/entities.py @@ -1220,6 +1220,8 @@ def get_archived_representations( as filter. Filter ignored if 'None' is passed. version_ids (Iterable[str]): Subset ids used as parent filter. Filter ignored if 'None' is passed. + context_filters (Dict[str, List[str, re.Pattern]]): Filter by + representation context fields. names_by_version_ids (dict[ObjectId, List[str]]): Complex filtering using version ids and list of names under the version. fields (Iterable[str]): Fields that should be returned. All fields are From aefb992ce55145f94790bbaa5cdbf17136684e1e Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 12 Aug 2022 13:55:35 +0200 Subject: [PATCH 27/28] removed unused 'is_context' property --- .../workfile/abstract_template_loader.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/openpype/pipeline/workfile/abstract_template_loader.py b/openpype/pipeline/workfile/abstract_template_loader.py index 1c8ede25e6..e2f9fdba0f 100644 --- a/openpype/pipeline/workfile/abstract_template_loader.py +++ b/openpype/pipeline/workfile/abstract_template_loader.py @@ -465,23 +465,6 @@ class AbstractPlaceholder: return self.data["loader"] - @property - def is_context(self): - """Check if is placeholder context type. - - context_asset: For loading current asset - linked_asset: For loading linked assets - - Question: - There seems to be more build options and this property is not used, - should be removed? - - Returns: - bool: true if placeholder is a context placeholder - """ - - return self.builder_type == "context_asset" - @property def is_valid(self): """Test validity of placeholder. From 32c2440e4a6d5d5ec3d54c2d1a44ffc5f0f81ae5 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 12 Aug 2022 13:55:55 +0200 Subject: [PATCH 28/28] fix docstring header --- openpype/pipeline/workfile/abstract_template_loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/pipeline/workfile/abstract_template_loader.py b/openpype/pipeline/workfile/abstract_template_loader.py index e2f9fdba0f..05a98a1ddc 100644 --- a/openpype/pipeline/workfile/abstract_template_loader.py +++ b/openpype/pipeline/workfile/abstract_template_loader.py @@ -456,7 +456,7 @@ class AbstractPlaceholder: @property def loader_name(self): - """Return placeholder loader type. + """Return placeholder loader name. Returns: str: Loader name that will be used to load placeholder