From 13d3b42617df5765fb9aeb56319f2553ebe2c900 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 18 Mar 2021 18:39:01 +0100 Subject: [PATCH 01/25] store only application name to config --- pype/modules/ftrack/lib/avalon_sync.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pype/modules/ftrack/lib/avalon_sync.py b/pype/modules/ftrack/lib/avalon_sync.py index d639e814a5..e4631aad29 100644 --- a/pype/modules/ftrack/lib/avalon_sync.py +++ b/pype/modules/ftrack/lib/avalon_sync.py @@ -194,12 +194,8 @@ def get_project_apps(in_app_list): missing_app_msg = "Missing definition of application" application_manager = ApplicationManager() for app_name in in_app_list: - app = application_manager.applications.get(app_name) - if app: - apps.append({ - "name": app_name, - "label": app.full_label - }) + if application_manager.applications.get(app_name): + apps.append({"name": app_name}) else: warnings[missing_app_msg].append(app_name) return apps, warnings From 834aa9c4053d0e59a6fbb3d9de738bce3215e436 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 18 Mar 2021 18:39:29 +0100 Subject: [PATCH 02/25] skip applications lookup if passed value is empty --- pype/modules/ftrack/lib/avalon_sync.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pype/modules/ftrack/lib/avalon_sync.py b/pype/modules/ftrack/lib/avalon_sync.py index e4631aad29..be1e150cb7 100644 --- a/pype/modules/ftrack/lib/avalon_sync.py +++ b/pype/modules/ftrack/lib/avalon_sync.py @@ -191,6 +191,9 @@ def get_project_apps(in_app_list): apps = [] warnings = collections.defaultdict(list) + if not in_app_list: + return apps, warnings + missing_app_msg = "Missing definition of application" application_manager = ApplicationManager() for app_name in in_app_list: From f4e11d69e3fb176aed2729a915a09b99ee6ac8a9 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 18 Mar 2021 18:39:38 +0100 Subject: [PATCH 03/25] pop applicaitons from data --- pype/modules/ftrack/lib/avalon_sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/modules/ftrack/lib/avalon_sync.py b/pype/modules/ftrack/lib/avalon_sync.py index be1e150cb7..5ce6263198 100644 --- a/pype/modules/ftrack/lib/avalon_sync.py +++ b/pype/modules/ftrack/lib/avalon_sync.py @@ -1140,7 +1140,7 @@ class SyncEntitiesFactory: proj_schema = entity["project_schema"] task_types = proj_schema["_task_type_schema"]["types"] proj_apps, warnings = get_project_apps( - (data.get("applications") or []) + data.pop("applications", []) ) for msg, items in warnings.items(): if not msg or not items: From 166f623cd483bddd4baf6719e8b30a33f2947372 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 18 Mar 2021 18:39:50 +0100 Subject: [PATCH 04/25] don't care about old parent name on rename --- pype/modules/ftrack/lib/avalon_sync.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pype/modules/ftrack/lib/avalon_sync.py b/pype/modules/ftrack/lib/avalon_sync.py index 5ce6263198..bab69ab8e4 100644 --- a/pype/modules/ftrack/lib/avalon_sync.py +++ b/pype/modules/ftrack/lib/avalon_sync.py @@ -1427,8 +1427,13 @@ class SyncEntitiesFactory: old_parent_name = self.entities_dict[ self.ft_project_id]["name"] else: - old_parent_name = self.avalon_ents_by_id[ - ftrack_parent_mongo_id]["name"] + old_parent_name = "N/A" + if ftrack_parent_mongo_id in self.avalon_ents_by_id: + old_parent_name = ( + self.avalon_ents_by_id + [ftrack_parent_mongo_id] + ["name"] + ) self.updates[avalon_id]["data"] = { "visualParent": new_parent_id From b166b8133a809966ae387c42766ec5dfc8abdd7b Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 18 Mar 2021 18:40:06 +0100 Subject: [PATCH 05/25] task types are not overriden with new but only added new ones --- pype/modules/ftrack/lib/avalon_sync.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/pype/modules/ftrack/lib/avalon_sync.py b/pype/modules/ftrack/lib/avalon_sync.py index bab69ab8e4..7a8800ffeb 100644 --- a/pype/modules/ftrack/lib/avalon_sync.py +++ b/pype/modules/ftrack/lib/avalon_sync.py @@ -2143,11 +2143,22 @@ class SyncEntitiesFactory: final_doc_data = self.entities_dict[self.ft_project_id]["final_entity"] final_doc_tasks = final_doc_data["config"].pop("tasks") current_doc_tasks = self.avalon_project.get("config", {}).get("tasks") - # Update project's tasks if tasks are empty or are not same - if not final_doc_tasks: + # Update project's task types + if not current_doc_tasks: update_tasks = True else: - update_tasks = final_doc_tasks != current_doc_tasks + # Check if task types are same + update_tasks = False + for task_type in final_doc_tasks: + if task_type not in current_doc_tasks: + update_tasks = True + break + + # Update new task types + # - but keep data about existing types and only add new one + if update_tasks: + for task_type, type_data in current_doc_tasks.items(): + final_doc_tasks[task_type] = type_data changes = self.compare_dict(final_doc_data, self.avalon_project) From 19ab249c6ddd1d6f1e88e6cee3bb345e30dd5850 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 18 Mar 2021 19:03:28 +0100 Subject: [PATCH 06/25] applications are not stored to data in sync to avalon event --- .../event_sync_to_avalon.py | 40 ++++++++++--------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py b/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py index 7c9c4d196f..a9e1f4282d 100644 --- a/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py +++ b/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py @@ -1822,6 +1822,27 @@ class SyncToAvalonEvent(BaseEvent): ent_cust_attrs = [] for key, values in ent_info["changes"].items(): + if entType == "show" and key == "applications": + # Store apps to project't config + apps_str = ent_info["changes"]["applications"]["new"] + cust_attr_apps = [ + app_name.strip() + for app_name in apps_str.split(", ") if app_name + ] + + proj_apps, warnings = ( + avalon_sync.get_project_apps(cust_attr_apps) + ) + if "config" not in self.updates[mongo_id]: + self.updates[mongo_id]["config"] = {} + self.updates[mongo_id]["config"]["apps"] = proj_apps + + for msg, items in warnings.items(): + if not msg or not items: + continue + self.report_items["warning"][msg] = items + continue + if key in hier_attrs_keys: self.hier_cust_attrs_changes[key].append(ftrack_id) continue @@ -1839,25 +1860,6 @@ class SyncToAvalonEvent(BaseEvent): ) ) - if entType != "show" or key != "applications": - continue - - # Store apps to project't config - apps_str = ent_info["changes"]["applications"]["new"] - cust_attr_apps = [app for app in apps_str.split(", ") if app] - - proj_apps, warnings = ( - avalon_sync.get_project_apps(cust_attr_apps) - ) - if "config" not in self.updates[mongo_id]: - self.updates[mongo_id]["config"] = {} - self.updates[mongo_id]["config"]["apps"] = proj_apps - - for msg, items in warnings.items(): - if not msg or not items: - continue - self.report_items["warning"][msg] = items - def process_hier_cleanup(self): if ( not self.moved_in_avalon and From 544869cd314b26f6b74c071e5ef871097f405dc1 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 19 Mar 2021 13:27:30 +0100 Subject: [PATCH 07/25] it is possible to define keys to query for custom attributes --- pype/modules/ftrack/lib/avalon_sync.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/pype/modules/ftrack/lib/avalon_sync.py b/pype/modules/ftrack/lib/avalon_sync.py index 7a8800ffeb..f7feec7475 100644 --- a/pype/modules/ftrack/lib/avalon_sync.py +++ b/pype/modules/ftrack/lib/avalon_sync.py @@ -83,15 +83,23 @@ def check_regex(name, entity_type, in_schema=None, schema_patterns=None): return False -def get_pype_attr(session, split_hierarchical=True): +def get_pype_attr(session, split_hierarchical=True, query_keys=None): custom_attributes = [] hier_custom_attributes = [] + if not query_keys: + query_keys = [ + "id", + "entity_type", + "object_type_id", + "is_hierarchical", + "default" + ] # TODO remove deprecated "avalon" group from query cust_attrs_query = ( - "select id, entity_type, object_type_id, is_hierarchical, default" + "select {}" " from CustomAttributeConfiguration" - " where group.name in (\"avalon\", \"pype\")" - ) + " where group.name in (\"avalon\", \"{}\")" + ).format(join_query_keys(query_keys), CUST_ATTR_GROUP) all_avalon_attr = session.query(cust_attrs_query).all() for cust_attr in all_avalon_attr: if split_hierarchical and cust_attr["is_hierarchical"]: From 21ba1896a218aa8cb7d6c4ae60a9a81f6eebf77c Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 19 Mar 2021 13:27:39 +0100 Subject: [PATCH 08/25] implemented join query keys --- pype/modules/ftrack/lib/avalon_sync.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pype/modules/ftrack/lib/avalon_sync.py b/pype/modules/ftrack/lib/avalon_sync.py index f7feec7475..f0306c2073 100644 --- a/pype/modules/ftrack/lib/avalon_sync.py +++ b/pype/modules/ftrack/lib/avalon_sync.py @@ -83,6 +83,10 @@ def check_regex(name, entity_type, in_schema=None, schema_patterns=None): return False +def join_query_keys(keys): + return ",".join(["\"{}\"".format(key) for key in keys]) + + def get_pype_attr(session, split_hierarchical=True, query_keys=None): custom_attributes = [] hier_custom_attributes = [] From 4408137bdef5ab7ebd218a8921f0e47238dfdeb4 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 19 Mar 2021 14:53:42 +0100 Subject: [PATCH 09/25] fix key query --- pype/modules/ftrack/lib/avalon_sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/modules/ftrack/lib/avalon_sync.py b/pype/modules/ftrack/lib/avalon_sync.py index f0306c2073..addc8a4a9c 100644 --- a/pype/modules/ftrack/lib/avalon_sync.py +++ b/pype/modules/ftrack/lib/avalon_sync.py @@ -103,7 +103,7 @@ def get_pype_attr(session, split_hierarchical=True, query_keys=None): "select {}" " from CustomAttributeConfiguration" " where group.name in (\"avalon\", \"{}\")" - ).format(join_query_keys(query_keys), CUST_ATTR_GROUP) + ).format(", ".join(query_keys), CUST_ATTR_GROUP) all_avalon_attr = session.query(cust_attrs_query).all() for cust_attr in all_avalon_attr: if split_hierarchical and cust_attr["is_hierarchical"]: From 85a03bd2d7f41dddbeeb6e367d6760a65dd455f1 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 19 Mar 2021 14:54:22 +0100 Subject: [PATCH 10/25] implemented method that can convert changes string to value --- .../event_sync_to_avalon.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py b/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py index a9e1f4282d..443e428c01 100644 --- a/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py +++ b/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py @@ -1,6 +1,7 @@ import os import collections import copy +import json import queue import time import datetime @@ -10,6 +11,7 @@ import traceback from bson.objectid import ObjectId from pymongo import UpdateOne +import arrow import ftrack_api from avalon import schema @@ -1860,6 +1862,41 @@ class SyncToAvalonEvent(BaseEvent): ) ) + def convert_value_by_cust_attr_conf(self, value, cust_attr_conf): + type_id = cust_attr_conf["type_id"] + cust_attr_type_name = self.cust_attr_types_by_id[type_id]["name"] + ignored = ( + "expression", "notificationtype", "dynamic enumerator" + ) + if cust_attr_type_name in ignored: + return None + + if cust_attr_type_name == "text": + return value + + if cust_attr_type_name == "boolean": + if value == "1": + return True + if value == "0": + return False + return bool(value) + + if cust_attr_type_name == "date": + return arrow.get(value) + + cust_attr_config = json.loads(cust_attr_conf["config"]) + + if cust_attr_type_name == "number": + if cust_attr_config["isdecimal"]: + return float(value) + return int(value) + + if cust_attr_type_name == "enumerator": + if not cust_attr_config["multiSelect"]: + return value + return value.split(", ") + return value + def process_hier_cleanup(self): if ( not self.moved_in_avalon and From d40165ef946993d836837dd5e4bea81ac1a0650a Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 19 Mar 2021 14:55:02 +0100 Subject: [PATCH 11/25] removed duplicated property avalon_custom_attributes --- .../event_handlers_server/event_sync_to_avalon.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py b/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py index 443e428c01..497edf1ae4 100644 --- a/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py +++ b/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py @@ -229,15 +229,6 @@ class SyncToAvalonEvent(BaseEvent): return self._changeability_by_mongo_id - @property - def avalon_custom_attributes(self): - """Return info about changeability of entity and it's parents.""" - if self._avalon_custom_attributes is None: - self._avalon_custom_attributes = avalon_sync.get_pype_attr( - self.process_session - ) - return self._avalon_custom_attributes - def remove_cached_by_key(self, key, values): if self._avalon_ents is None: return @@ -393,7 +384,6 @@ class SyncToAvalonEvent(BaseEvent): self._avalon_archived_by_id = None self._avalon_archived_by_name = None - self._avalon_custom_attributes = None self._ent_types_by_name = None self.ftrack_ents_by_id = {} @@ -1238,7 +1228,7 @@ class SyncToAvalonEvent(BaseEvent): def get_cust_attr_values(self, entity, keys=None): output = {} - custom_attrs, hier_attrs = self.avalon_custom_attributes + custom_attrs, hier_attrs = self.avalon_cust_attrs not_processed_keys = True if keys: not_processed_keys = [k for k in keys] From 40865b992af3352358574ff92f5dc56b814b9193 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 19 Mar 2021 14:55:14 +0100 Subject: [PATCH 12/25] define query keys of custom attributes --- .../event_handlers_server/event_sync_to_avalon.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py b/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py index 497edf1ae4..8f53d61781 100644 --- a/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py +++ b/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py @@ -33,6 +33,15 @@ class SyncToAvalonEvent(BaseEvent): ignore_ent_types = ["Milestone"] ignore_keys = ["statusid", "thumbid"] + cust_attr_query_keys = [ + "id", + "key", + "entity_type", + "object_type_id", + "is_hierarchical", + "config", + "default" + ] project_query = ( "select full_name, name, custom_attributes" ", project_schema._task_type_schema.types.name" @@ -117,7 +126,7 @@ class SyncToAvalonEvent(BaseEvent): def avalon_cust_attrs(self): if self._avalon_cust_attrs is None: self._avalon_cust_attrs = avalon_sync.get_pype_attr( - self.process_session + self.process_session, query_keys=self.cust_attr_query_keys ) return self._avalon_cust_attrs From 1c7010b0cbf87d1a45cdbdce9c3de3389e297ed9 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 19 Mar 2021 14:55:43 +0100 Subject: [PATCH 13/25] implemented property cust_attr_types_by_id --- .../event_handlers_server/event_sync_to_avalon.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py b/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py index 8f53d61781..2b6f2e33df 100644 --- a/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py +++ b/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py @@ -130,6 +130,18 @@ class SyncToAvalonEvent(BaseEvent): ) return self._avalon_cust_attrs + @property + def cust_attr_types_by_id(self): + if self._cust_attr_types_by_id is None: + cust_attr_types = self.process_session.query( + "select id, name from CustomAttributeType" + ).all() + self._cust_attr_types_by_id = { + cust_attr_type["id"]: cust_attr_type + for cust_attr_type in cust_attr_types + } + return self._cust_attr_types_by_id + @property def avalon_entities(self): if self._avalon_ents is None: @@ -382,6 +394,7 @@ class SyncToAvalonEvent(BaseEvent): self._cur_project = None self._avalon_cust_attrs = None + self._cust_attr_types_by_id = None self._avalon_ents = None self._avalon_ents_by_id = None From 0231c73cf773d2903cec1f273af7436e76858429 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 19 Mar 2021 14:55:50 +0100 Subject: [PATCH 14/25] fixed ordered dict --- .../ftrack/event_handlers_server/event_sync_to_avalon.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py b/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py index 2b6f2e33df..cf5b9d4e26 100644 --- a/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py +++ b/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py @@ -1562,10 +1562,9 @@ class SyncToAvalonEvent(BaseEvent): ).format(entity_type, ent_info["entityType"])) continue - _entity_key = collections.OrderedDict({ - "configuration_id": mongo_id_configuration_id, - "entity_id": ftrack_id - }) + _entity_key = collections.OrderedDict() + _entity_key["configuration_id"] = mongo_id_configuration_id + _entity_key["entity_id"] = ftrack_id self.process_session.recorded_operations.push( ftrack_api.operation.UpdateEntityOperation( From a6b8300c32a369eb7278660fb3d31d9b3e00a2db Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 19 Mar 2021 14:56:10 +0100 Subject: [PATCH 15/25] event value is converted to real value --- .../event_sync_to_avalon.py | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py b/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py index cf5b9d4e26..fec2d672cf 100644 --- a/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py +++ b/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py @@ -1803,6 +1803,10 @@ class SyncToAvalonEvent(BaseEvent): return cust_attrs, hier_attrs = self.avalon_cust_attrs + hier_attrs_by_key = { + attr["key"]: attr + for attr in hier_attrs + } cust_attrs_by_obj_id = collections.defaultdict(dict) for cust_attr in cust_attrs: key = cust_attr["key"] @@ -1818,8 +1822,6 @@ class SyncToAvalonEvent(BaseEvent): obj_id = cust_attr["object_type_id"] cust_attrs_by_obj_id[obj_id][key] = cust_attr - hier_attrs_keys = [attr["key"] for attr in hier_attrs] - for ftrack_id, ent_info in ent_infos.items(): mongo_id = ftrack_mongo_mapping[ftrack_id] entType = ent_info["entityType"] @@ -1832,19 +1834,25 @@ class SyncToAvalonEvent(BaseEvent): # Ftrack's entity_type does not have defined custom attributes if ent_cust_attrs is None: - ent_cust_attrs = [] + ent_cust_attrs = {} for key, values in ent_info["changes"].items(): + if key in hier_attrs_by_key: + self.hier_cust_attrs_changes[key].append(ftrack_id) + continue + + if key not in ent_cust_attrs: + continue + + value = values["new"] + new_value = self.convert_value_by_cust_attr_conf( + value, ent_cust_attrs[key] + ) + if entType == "show" and key == "applications": # Store apps to project't config - apps_str = ent_info["changes"]["applications"]["new"] - cust_attr_apps = [ - app_name.strip() - for app_name in apps_str.split(", ") if app_name - ] - proj_apps, warnings = ( - avalon_sync.get_project_apps(cust_attr_apps) + avalon_sync.get_project_apps(new_value) ) if "config" not in self.updates[mongo_id]: self.updates[mongo_id]["config"] = {} @@ -1856,20 +1864,12 @@ class SyncToAvalonEvent(BaseEvent): self.report_items["warning"][msg] = items continue - if key in hier_attrs_keys: - self.hier_cust_attrs_changes[key].append(ftrack_id) - continue - - if key not in ent_cust_attrs: - continue - if "data" not in self.updates[mongo_id]: self.updates[mongo_id]["data"] = {} - value = values["new"] - self.updates[mongo_id]["data"][key] = value + self.updates[mongo_id]["data"][key] = new_value self.log.debug( "Setting data value of \"{}\" to \"{}\" <{}>".format( - key, value, ent_path + key, new_value, ent_path ) ) From 2e569e70af673458bd33bc75f9e7986117238b49 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 19 Mar 2021 14:56:21 +0100 Subject: [PATCH 16/25] use join function --- .../ftrack/event_handlers_server/event_sync_to_avalon.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py b/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py index fec2d672cf..c1c0ac0032 100644 --- a/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py +++ b/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py @@ -2137,16 +2137,12 @@ class SyncToAvalonEvent(BaseEvent): parent_queue.put(parent_ent) # Prepare values to query - entity_ids_joined = ", ".join([ - "\"{}\"".format(id) for id in cust_attrs_ftrack_ids - ]) configuration_ids = set() for key in hier_cust_attrs_keys: configuration_ids.add(hier_attr_id_by_key[key]) - attributes_joined = ", ".join([ - "\"{}\"".format(conf_id) for conf_id in configuration_ids - ]) + entity_ids_joined = self.join_query_keys(cust_attrs_ftrack_ids) + attributes_joined = self.join_query_keys(configuration_ids) queries = [{ "action": "query", From e98fe26aa8fe0223c309d4361ea56d884527fc30 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 19 Mar 2021 14:56:27 +0100 Subject: [PATCH 17/25] formatting changes --- .../ftrack/event_handlers_server/event_sync_to_avalon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py b/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py index c1c0ac0032..d71a94aabf 100644 --- a/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py +++ b/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py @@ -2018,7 +2018,7 @@ class SyncToAvalonEvent(BaseEvent): self.update_entities() return - cust_attrs, hier_attrs = self.avalon_cust_attrs + _, hier_attrs = self.avalon_cust_attrs # Hierarchical custom attributes preparation *** hier_attr_key_by_id = { From c502cc2d1c58384074ce50a8f2599e210ad2203a Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 19 Mar 2021 15:15:13 +0100 Subject: [PATCH 18/25] event sync convert hierarchical values to proper type --- .../event_sync_to_avalon.py | 44 ++++++++++++++++++- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py b/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py index d71a94aabf..ba60945968 100644 --- a/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py +++ b/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py @@ -2159,10 +2159,43 @@ class SyncToAvalonEvent(BaseEvent): ftrack_project_id = self.cur_project["id"] + attr_types_by_id = self.cust_attr_types_by_id + convert_types_by_id = {} for attr in hier_attrs: key = attr["key"] if key not in hier_cust_attrs_keys: continue + + type_id = attr["type_id"] + attr_id = attr["id"] + cust_attr_type_name = attr_types_by_id[type_id]["name"] + convert_type = None + if cust_attr_type_name == "text": + convert_type = str + + elif cust_attr_type_name == "boolean": + convert_type = bool + + elif cust_attr_type_name in ( + "date", "expression", "notificationtype", "dynamic enumerator" + ): + pass + + else: + cust_attr_config = json.loads(attr["config"]) + if cust_attr_type_name == "number": + if cust_attr_config["isdecimal"]: + convert_type = float + else: + convert_type = int + + elif cust_attr_type_name == "enumerator": + if cust_attr_config["multiSelect"]: + convert_type = list + else: + convert_type = str + + convert_types_by_id[attr_id] = convert_type entities_dict[ftrack_project_id]["hier_attrs"][key] = ( attr["default"] ) @@ -2173,8 +2206,15 @@ class SyncToAvalonEvent(BaseEvent): if value["value"] is None: continue entity_id = value["entity_id"] - key = hier_attr_key_by_id[value["configuration_id"]] - entities_dict[entity_id]["hier_attrs"][key] = value["value"] + configuration_id = value["configuration_id"] + + convert_type = convert_types_by_id[configuration_id] + key = hier_attr_key_by_id[configuration_id] + + the_value = value["value"] + if convert_type: + the_value = convert_type(the_value) + entities_dict[entity_id]["hier_attrs"][key] = the_value # Get dictionary with not None hierarchical values to pull to childs project_values = {} From 47a7fbe18130359a028946a22e461afb6838dba7 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 19 Mar 2021 15:26:41 +0100 Subject: [PATCH 19/25] implemented function to retrieve python type for passed custom attribute --- pype/modules/ftrack/lib/avalon_sync.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/pype/modules/ftrack/lib/avalon_sync.py b/pype/modules/ftrack/lib/avalon_sync.py index addc8a4a9c..9aa76539b6 100644 --- a/pype/modules/ftrack/lib/avalon_sync.py +++ b/pype/modules/ftrack/lib/avalon_sync.py @@ -119,6 +119,31 @@ def get_pype_attr(session, split_hierarchical=True, query_keys=None): return custom_attributes +def get_python_type_for_custom_attribute(cust_attr, cust_attr_type_name=None): + if cust_attr_type_name is None: + cust_attr_type_name = cust_attr["type"]["name"] + + if cust_attr_type_name == "text": + return str + + if cust_attr_type_name == "boolean": + return bool + + if cust_attr_type_name in ("number", "enumerator"): + cust_attr_config = json.loads(cust_attr["config"]) + if cust_attr_type_name == "number": + if cust_attr_config["isdecimal"]: + return float + return int + + if cust_attr_type_name == "enumerator": + if cust_attr_config["multiSelect"]: + return list + return str + # "date", "expression", "notificationtype", "dynamic enumerator" + return None + + def from_dict_to_set(data, is_project): """ Converts 'data' into $set part of MongoDB update command. From a91c0368e2de1ac0d8aed746ed544bf301269cef Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 19 Mar 2021 15:27:01 +0100 Subject: [PATCH 20/25] event sync to avalon is using get_python_type_for_custom_attribute --- .../event_sync_to_avalon.py | 28 ++----------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py b/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py index ba60945968..c03abbd52f 100644 --- a/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py +++ b/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py @@ -2169,31 +2169,9 @@ class SyncToAvalonEvent(BaseEvent): type_id = attr["type_id"] attr_id = attr["id"] cust_attr_type_name = attr_types_by_id[type_id]["name"] - convert_type = None - if cust_attr_type_name == "text": - convert_type = str - - elif cust_attr_type_name == "boolean": - convert_type = bool - - elif cust_attr_type_name in ( - "date", "expression", "notificationtype", "dynamic enumerator" - ): - pass - - else: - cust_attr_config = json.loads(attr["config"]) - if cust_attr_type_name == "number": - if cust_attr_config["isdecimal"]: - convert_type = float - else: - convert_type = int - - elif cust_attr_type_name == "enumerator": - if cust_attr_config["multiSelect"]: - convert_type = list - else: - convert_type = str + convert_type = avalon_sync.get_python_type_for_custom_attribute( + attr, cust_attr_type_name + ) convert_types_by_id[attr_id] = convert_type entities_dict[ftrack_project_id]["hier_attrs"][key] = ( From f0f4a1c65bb6b00df5952c8eb07dcaba34cdde70 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 19 Mar 2021 15:38:55 +0100 Subject: [PATCH 21/25] define query keys --- pype/modules/ftrack/lib/avalon_sync.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pype/modules/ftrack/lib/avalon_sync.py b/pype/modules/ftrack/lib/avalon_sync.py index 9aa76539b6..d75a6babdd 100644 --- a/pype/modules/ftrack/lib/avalon_sync.py +++ b/pype/modules/ftrack/lib/avalon_sync.py @@ -321,6 +321,16 @@ def get_hierarchical_attributes(session, entity, attr_names, attr_defaults={}): class SyncEntitiesFactory: dbcon = AvalonMongoDB() + cust_attr_query_keys = [ + "id", + "key", + "entity_type", + "object_type_id", + "is_hierarchical", + "config", + "default" + ] + project_query = ( "select full_name, name, custom_attributes" ", project_schema._task_type_schema.types.name" @@ -866,7 +876,9 @@ class SyncEntitiesFactory: def set_cutom_attributes(self): self.log.debug("* Preparing custom attributes") # Get custom attributes and values - custom_attrs, hier_attrs = get_pype_attr(self.session) + custom_attrs, hier_attrs = get_pype_attr( + self.session, query_keys=self.cust_attr_query_keys + ) ent_types = self.session.query("select id, name from ObjectType").all() ent_types_by_name = { ent_type["name"]: ent_type["id"] for ent_type in ent_types From c8bc66ae9ed6a220ed5c3657bce6698c190cdf65 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 19 Mar 2021 15:41:21 +0100 Subject: [PATCH 22/25] sync to avalon action is also using proper value type --- pype/modules/ftrack/lib/avalon_sync.py | 63 ++++++++++++++++++++++---- 1 file changed, 54 insertions(+), 9 deletions(-) diff --git a/pype/modules/ftrack/lib/avalon_sync.py b/pype/modules/ftrack/lib/avalon_sync.py index d75a6babdd..2db124235e 100644 --- a/pype/modules/ftrack/lib/avalon_sync.py +++ b/pype/modules/ftrack/lib/avalon_sync.py @@ -883,6 +883,14 @@ class SyncEntitiesFactory: ent_types_by_name = { ent_type["name"]: ent_type["id"] for ent_type in ent_types } + # Custom attribute types + cust_attr_types = self.session.query( + "select id, name from CustomAttributeType" + ).all() + cust_attr_type_name_by_id = { + cust_attr_type["id"]: cust_attr_type["name"] + for cust_attr_type in cust_attr_types + } # store default values per entity type attrs_per_entity_type = collections.defaultdict(dict) @@ -892,9 +900,20 @@ class SyncEntitiesFactory: avalon_attrs_ca_id = collections.defaultdict(dict) attribute_key_by_id = {} + convert_types_by_attr_id = {} for cust_attr in custom_attrs: key = cust_attr["key"] - attribute_key_by_id[cust_attr["id"]] = key + attr_id = cust_attr["id"] + type_id = cust_attr["type_id"] + + attribute_key_by_id[attr_id] = key + cust_attr_type_name = cust_attr_type_name_by_id[type_id] + + convert_type = get_python_type_for_custom_attribute( + cust_attr, cust_attr_type_name + ) + convert_types_by_attr_id[attr_id] = convert_type + ca_ent_type = cust_attr["entity_type"] if key.startswith("avalon_"): if ca_ent_type == "show": @@ -988,24 +1007,44 @@ class SyncEntitiesFactory: for item in values["data"]: entity_id = item["entity_id"] - key = attribute_key_by_id[item["configuration_id"]] + attr_id = item["configuration_id"] + key = attribute_key_by_id[attr_id] store_key = "custom_attributes" if key.startswith("avalon_"): store_key = "avalon_attrs" - self.entities_dict[entity_id][store_key][key] = item["value"] + + convert_type = convert_types_by_attr_id[attr_id] + value = item["value"] + if convert_type: + value = convert_type(value) + self.entities_dict[entity_id][store_key][key] = value # process hierarchical attributes - self.set_hierarchical_attribute(hier_attrs, sync_ids) + self.set_hierarchical_attribute( + hier_attrs, sync_ids, cust_attr_type_name_by_id + ) - def set_hierarchical_attribute(self, hier_attrs, sync_ids): + def set_hierarchical_attribute( + self, hier_attrs, sync_ids, cust_attr_type_name_by_id + ): # collect all hierarchical attribute keys # and prepare default values to project attributes_by_key = {} attribute_key_by_id = {} + convert_types_by_attr_id = {} for attr in hier_attrs: key = attr["key"] - attribute_key_by_id[attr["id"]] = key + attr_id = attr["id"] + type_id = attr["type_id"] + attribute_key_by_id[attr_id] = key attributes_by_key[key] = attr + + cust_attr_type_name = cust_attr_type_name_by_id[type_id] + convert_type = get_python_type_for_custom_attribute( + attr, cust_attr_type_name + ) + convert_types_by_attr_id[attr_id] = convert_type + self.hier_cust_attr_ids_by_key[key] = attr["id"] store_key = "hier_attrs" @@ -1040,7 +1079,7 @@ class SyncEntitiesFactory: else: prepare_dict[key] = None - for id, entity_dict in self.entities_dict.items(): + for entity_dict in self.entities_dict.values(): # Skip project because has stored defaults at the moment if entity_dict["entity_type"] == "project": continue @@ -1078,8 +1117,14 @@ class SyncEntitiesFactory: or (isinstance(value, (tuple, list)) and not value) ): continue + + attr_id = item["configuration_id"] + convert_type = convert_types_by_attr_id[attr_id] + if convert_type: + value = convert_type(value) + entity_id = item["entity_id"] - key = attribute_key_by_id[item["configuration_id"]] + key = attribute_key_by_id[attr_id] if key.startswith("avalon_"): store_key = "avalon_attrs" avalon_hier.append(key) @@ -2436,7 +2481,7 @@ class SyncEntitiesFactory: if new_entity_id not in p_chilren: self.entities_dict[parent_id]["children"].append(new_entity_id) - cust_attr, hier_attrs = get_pype_attr(self.session) + cust_attr, _ = get_pype_attr(self.session) for _attr in cust_attr: key = _attr["key"] if key not in av_entity["data"]: From c8f1e14bb8453dbb845b1169bb44dcf2c441374b Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 19 Mar 2021 15:43:26 +0100 Subject: [PATCH 23/25] added some docstrings --- pype/modules/ftrack/lib/avalon_sync.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pype/modules/ftrack/lib/avalon_sync.py b/pype/modules/ftrack/lib/avalon_sync.py index 2db124235e..ff305fe17b 100644 --- a/pype/modules/ftrack/lib/avalon_sync.py +++ b/pype/modules/ftrack/lib/avalon_sync.py @@ -120,6 +120,14 @@ def get_pype_attr(session, split_hierarchical=True, query_keys=None): def get_python_type_for_custom_attribute(cust_attr, cust_attr_type_name=None): + """Python type that should value of custom attribute have. + + This function is mainly for number type which is always float from ftrack. + + Returns: + type: Python type which call be called on object to convert the object + to the type or None if can't figure out. + """ if cust_attr_type_name is None: cust_attr_type_name = cust_attr["type"]["name"] From df12e9a112fa6c17d180238ec9d7525f100d4c21 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 19 Mar 2021 16:13:47 +0100 Subject: [PATCH 24/25] fixed and made faster new entity creation --- .../event_sync_to_avalon.py | 59 +++------ pype/modules/ftrack/lib/avalon_sync.py | 125 +++++++++--------- 2 files changed, 81 insertions(+), 103 deletions(-) diff --git a/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py b/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py index c03abbd52f..4b0b96f166 100644 --- a/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py +++ b/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py @@ -1248,48 +1248,18 @@ class SyncToAvalonEvent(BaseEvent): return final_entity - def get_cust_attr_values(self, entity, keys=None): + def get_cust_attr_values(self, entity): output = {} custom_attrs, hier_attrs = self.avalon_cust_attrs - not_processed_keys = True - if keys: - not_processed_keys = [k for k in keys] + # Notmal custom attributes - processed_keys = [] for attr in custom_attrs: - if not not_processed_keys: - break key = attr["key"] - if key in processed_keys: - continue + if key in entity["custom_attributes"]: + output[key] = entity["custom_attributes"][key] - if key not in entity["custom_attributes"]: - continue - - if keys: - if key not in keys: - continue - else: - not_processed_keys.remove(key) - - output[key] = entity["custom_attributes"][key] - processed_keys.append(key) - - if not not_processed_keys: - return output - - # Hierarchical cust attrs - hier_keys = [] - defaults = {} - for attr in hier_attrs: - key = attr["key"] - if keys and key not in keys: - continue - hier_keys.append(key) - defaults[key] = attr["default"] - - hier_values = avalon_sync.get_hierarchical_attributes( - self.process_session, entity, hier_keys, defaults + hier_values = avalon_sync.get_hierarchical_attributes_values( + self.process_session, entity, hier_attrs ) for key, val in hier_values.items(): if key == CUST_ATTR_ID_KEY: @@ -2147,7 +2117,8 @@ class SyncToAvalonEvent(BaseEvent): queries = [{ "action": "query", "expression": ( - "select value, entity_id from CustomAttributeValue " + "select value, entity_id, configuration_id" + " from CustomAttributeValue " "where entity_id in ({}) and configuration_id in ({})" ).format(entity_ids_joined, attributes_joined) }] @@ -2180,19 +2151,19 @@ class SyncToAvalonEvent(BaseEvent): # PREPARE DATA BEFORE THIS avalon_hier = [] - for value in values["data"]: - if value["value"] is None: + for item in values["data"]: + value = item["value"] + if value is None: continue - entity_id = value["entity_id"] - configuration_id = value["configuration_id"] + entity_id = item["entity_id"] + configuration_id = item["configuration_id"] convert_type = convert_types_by_id[configuration_id] key = hier_attr_key_by_id[configuration_id] - the_value = value["value"] if convert_type: - the_value = convert_type(the_value) - entities_dict[entity_id]["hier_attrs"][key] = the_value + value = convert_type(value) + entities_dict[entity_id]["hier_attrs"][key] = value # Get dictionary with not None hierarchical values to pull to childs project_values = {} diff --git a/pype/modules/ftrack/lib/avalon_sync.py b/pype/modules/ftrack/lib/avalon_sync.py index ff305fe17b..a6151bfba9 100644 --- a/pype/modules/ftrack/lib/avalon_sync.py +++ b/pype/modules/ftrack/lib/avalon_sync.py @@ -249,79 +249,84 @@ def get_project_apps(in_app_list): return apps, warnings -def get_hierarchical_attributes(session, entity, attr_names, attr_defaults={}): - entity_ids = [] - if entity.entity_type.lower() == "project": - entity_ids.append(entity["id"]) - else: - typed_context = session.query(( - "select ancestors.id, project from TypedContext where id is \"{}\"" - ).format(entity["id"])).one() - entity_ids.append(typed_context["id"]) - entity_ids.extend( - [ent["id"] for ent in reversed(typed_context["ancestors"])] +def get_hierarchical_attributes_values( + session, entity, hier_attrs, cust_attr_types=None +): + if not cust_attr_types: + cust_attr_types = session.query( + "select id, name from CustomAttributeType" + ).all() + + cust_attr_name_by_id = { + cust_attr_type["id"]: cust_attr_type["name"] + for cust_attr_type in cust_attr_types + } + # Hierarchical cust attrs + attr_key_by_id = {} + convert_types_by_attr_id = {} + defaults = {} + for attr in hier_attrs: + attr_id = attr["id"] + key = attr["key"] + type_id = attr["type_id"] + + attr_key_by_id[attr_id] = key + defaults[key] = attr["default"] + + cust_attr_type_name = cust_attr_name_by_id[type_id] + convert_type = get_python_type_for_custom_attribute( + attr, cust_attr_type_name ) - entity_ids.append(typed_context["project"]["id"]) + convert_types_by_attr_id[attr_id] = convert_type - missing_defaults = [] - for attr_name in attr_names: - if attr_name not in attr_defaults: - missing_defaults.append(attr_name) + entity_ids = [item["id"] for item in entity["link"]] + + join_ent_ids = join_query_keys(entity_ids) + join_attribute_ids = join_query_keys(attr_key_by_id.keys()) - join_ent_ids = ", ".join( - ["\"{}\"".format(entity_id) for entity_id in entity_ids] - ) - join_attribute_names = ", ".join( - ["\"{}\"".format(key) for key in attr_names] - ) queries = [] queries.append({ "action": "query", "expression": ( - "select value, entity_id from CustomAttributeValue " - "where entity_id in ({}) and configuration.key in ({})" - ).format(join_ent_ids, join_attribute_names) + "select value, configuration_id, entity_id" + " from CustomAttributeValue" + " where entity_id in ({}) and configuration_id in ({})" + ).format(join_ent_ids, join_attribute_ids) }) - if not missing_defaults: - if hasattr(session, "call"): - [values] = session.call(queries) - else: - [values] = session._call(queries) + if hasattr(session, "call"): + [values] = session.call(queries) else: - join_missing_names = ", ".join( - ["\"{}\"".format(key) for key in missing_defaults] - ) - queries.append({ - "action": "query", - "expression": ( - "select default from CustomAttributeConfiguration " - "where key in ({})" - ).format(join_missing_names) - }) - - [values, default_values] = session.call(queries) - for default_value in default_values: - key = default_value["data"][0]["key"] - attr_defaults[key] = default_value["data"][0]["default"] + [values] = session._call(queries) hier_values = {} - for key, val in attr_defaults.items(): + for key, val in defaults.items(): hier_values[key] = val if not values["data"]: return hier_values - _hier_values = collections.defaultdict(list) - for value in values["data"]: - key = value["configuration"]["key"] - _hier_values[key].append(value) + values_by_entity_id = collections.defaultdict(dict) + for item in values["data"]: + value = item["value"] + if value is None: + continue - for key, values in _hier_values.items(): - value = sorted( - values, key=lambda value: entity_ids.index(value["entity_id"]) - )[0] - hier_values[key] = value["value"] + attr_id = item["configuration_id"] + + convert_type = convert_types_by_attr_id[attr_id] + if convert_type: + value = convert_type(value) + + key = attr_key_by_id[attr_id] + entity_id = item["entity_id"] + values_by_entity_id[entity_id][key] = value + + for entity_id in entity_ids: + for key in attr_key_by_id.values(): + value = values_by_entity_id[entity_id].get(key) + if value is not None: + hier_values[key] = value return hier_values @@ -999,8 +1004,9 @@ class SyncEntitiesFactory: ]) cust_attr_query = ( - "select value, entity_id from ContextCustomAttributeValue " - "where entity_id in ({}) and configuration_id in ({})" + "select value, configuration_id, entity_id" + " from ContextCustomAttributeValue" + " where entity_id in ({}) and configuration_id in ({})" ) call_expr = [{ "action": "query", @@ -1106,8 +1112,9 @@ class SyncEntitiesFactory: call_expr = [{ "action": "query", "expression": ( - "select value, entity_id from ContextCustomAttributeValue " - "where entity_id in ({}) and configuration_id in ({})" + "select value, entity_id, configuration_id" + " from ContextCustomAttributeValue" + " where entity_id in ({}) and configuration_id in ({})" ).format(entity_ids_joined, attributes_joined) }] if hasattr(self.session, "call"): From 236ad6561783f861d25b56062fc95c4c02995634 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 19 Mar 2021 16:16:39 +0100 Subject: [PATCH 25/25] use precached values --- .../event_handlers_server/event_sync_to_avalon.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py b/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py index 4b0b96f166..be3a15b049 100644 --- a/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py +++ b/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py @@ -1259,13 +1259,17 @@ class SyncToAvalonEvent(BaseEvent): output[key] = entity["custom_attributes"][key] hier_values = avalon_sync.get_hierarchical_attributes_values( - self.process_session, entity, hier_attrs + self.process_session, + entity, + hier_attrs, + self.cust_attr_types_by_id ) for key, val in hier_values.items(): - if key == CUST_ATTR_ID_KEY: - continue output[key] = val + # Make sure mongo id is not set + output.pop(CUST_ATTR_ID_KEY, None) + return output def process_renamed(self):