From faa1d8028a5cfbfb73a2178bb7f22e2ab2bb6245 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 8 Oct 2020 13:32:06 +0200 Subject: [PATCH 01/13] filter tasks by entity_type and do task filtration at one place --- .../ftrack/events/event_sync_to_avalon.py | 34 +++++++------------ 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/pype/modules/ftrack/events/event_sync_to_avalon.py b/pype/modules/ftrack/events/event_sync_to_avalon.py index f3ef38b868..d021938606 100644 --- a/pype/modules/ftrack/events/event_sync_to_avalon.py +++ b/pype/modules/ftrack/events/event_sync_to_avalon.py @@ -555,8 +555,16 @@ class SyncToAvalonEvent(BaseEvent): ftrack_id = ftrack_id[0] # task modified, collect parent id of task, handle separately - if entityType.lower() == 'task': - self.modified_tasks_ftrackids.add(ent_info["parentId"]) + if entity_type.lower() == "task": + changes = ent_info.get("changes") or {} + if action == "move": + parent_changes = changes["parent_id"] + self.modified_tasks_ftrackids.add(parent_changes["new"]) + self.modified_tasks_ftrackids.add(parent_changes["old"]) + + elif "typeid" in changes or "name" in changes: + self.modified_tasks_ftrackids.add(ent_info["parentId"]) + continue if action == "move": ent_keys = ent_info["keys"] @@ -565,31 +573,15 @@ class SyncToAvalonEvent(BaseEvent): _ent_info = ent_info.copy() for ent_key in ent_keys: if ent_key == "parent_id": - # task parents modified, collect both - if entityType.lower() == 'task': - self.modified_tasks_ftrackids.add( - ent_info["changes"]["new"]) - self.modified_tasks_ftrackids.add( - ent_info["changes"]["old"]) _ent_info["changes"].pop(ent_key, None) _ent_info["keys"].remove(ent_key) else: ent_info["changes"].pop(ent_key, None) ent_info["keys"].remove(ent_key) - if entityType.lower() != 'task': - entities_by_action["update"][ftrack_id] = _ent_info - else: - if entityType.lower() == 'task': - self.modified_tasks_ftrackids.add( - ent_info["changes"]["parent_id"]["new"]) - self.modified_tasks_ftrackids.add( - ent_info["changes"]["parent_id"]["old"] - ) - + entities_by_action["update"][ftrack_id] = _ent_info # regular change process handles all other than Tasks - if entityType.lower() != 'task': - found_actions.add(action) - entities_by_action[action][ftrack_id] = ent_info + found_actions.add(action) + entities_by_action[action][ftrack_id] = ent_info found_actions = list(found_actions) if not found_actions and not self.modified_tasks_ftrackids: From 04c6fd33ebeb80c68a27c9ead8c5d58f7ca26a51 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 8 Oct 2020 13:32:27 +0200 Subject: [PATCH 02/13] fixed ftrack ids unin --- pype/modules/ftrack/events/event_sync_to_avalon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/modules/ftrack/events/event_sync_to_avalon.py b/pype/modules/ftrack/events/event_sync_to_avalon.py index d021938606..f7dc34054e 100644 --- a/pype/modules/ftrack/events/event_sync_to_avalon.py +++ b/pype/modules/ftrack/events/event_sync_to_avalon.py @@ -665,7 +665,7 @@ class SyncToAvalonEvent(BaseEvent): for action, _ftrack_ids in entities_by_action.items(): # skip updated (already prepared) and removed (not exist in ftrack) if action not in ("remove", "update"): - ftrack_ids.union(set(_ftrack_ids)) + ftrack_ids |= set(_ftrack_ids) # collect entity records data which might not be in event for entity in self._get_entities_for_ftrack_ids(ft_project["id"], From 3d0ab220c5c1fbc0eb740bf28e31e162de814daf Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 8 Oct 2020 13:32:56 +0200 Subject: [PATCH 03/13] do not use `_get_entities_for_ftrack_ids` for query as it find entities by parent id --- pype/modules/ftrack/events/event_sync_to_avalon.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pype/modules/ftrack/events/event_sync_to_avalon.py b/pype/modules/ftrack/events/event_sync_to_avalon.py index f7dc34054e..3dc0d92db2 100644 --- a/pype/modules/ftrack/events/event_sync_to_avalon.py +++ b/pype/modules/ftrack/events/event_sync_to_avalon.py @@ -668,9 +668,13 @@ class SyncToAvalonEvent(BaseEvent): ftrack_ids |= set(_ftrack_ids) # collect entity records data which might not be in event - for entity in self._get_entities_for_ftrack_ids(ft_project["id"], - ftrack_ids): - self.ftrack_ents_by_id[entity["id"]] = entity + if ftrack_ids: + joined_ids = ", ".join(["\"{}\"".format(id) for id in ftrack_ids]) + ftrack_entities = self.process_session.query( + self.entities_query_by_id.format(ft_project["id"], joined_ids) + ).all() + for entity in ftrack_entities: + self.ftrack_ents_by_id[entity["id"]] = entity # Filter updates where name is changing for ftrack_id, ent_info in updated.items(): From 06cf676bf8373855567f9920272d1253e1514a76 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 8 Oct 2020 13:33:19 +0200 Subject: [PATCH 04/13] check if there are unprocessed task changes before skipping whole process --- pype/modules/ftrack/events/event_sync_to_avalon.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pype/modules/ftrack/events/event_sync_to_avalon.py b/pype/modules/ftrack/events/event_sync_to_avalon.py index 3dc0d92db2..dd7ed1386f 100644 --- a/pype/modules/ftrack/events/event_sync_to_avalon.py +++ b/pype/modules/ftrack/events/event_sync_to_avalon.py @@ -623,9 +623,10 @@ class SyncToAvalonEvent(BaseEvent): # skip most of events where nothing has changed for avalon if ( - len(found_actions) == 1 and - found_actions[0] == "update" and - not updated + len(found_actions) == 1 + and found_actions[0] == "update" + and not updated + and not self.modified_tasks_ftrackids ): return True From a434041a27de61bbb0e2672d47f7c5958b4efc9f Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 8 Oct 2020 13:33:36 +0200 Subject: [PATCH 05/13] fixed undefined variable in remove --- pype/modules/ftrack/events/event_sync_to_avalon.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pype/modules/ftrack/events/event_sync_to_avalon.py b/pype/modules/ftrack/events/event_sync_to_avalon.py index dd7ed1386f..f2a94da56c 100644 --- a/pype/modules/ftrack/events/event_sync_to_avalon.py +++ b/pype/modules/ftrack/events/event_sync_to_avalon.py @@ -767,10 +767,10 @@ class SyncToAvalonEvent(BaseEvent): recreate_ents = [] removed_names = [] for ftrack_id, removed in ent_infos.items(): - if entity_type == "Task": + entity_type = removed["entity_type"] + if entity_type.lower() == "task": continue - entity_type = removed["entity_type"] removed_name = removed["changes"]["name"]["old"] avalon_ent = self.avalon_ents_by_ftrack_id.get(ftrack_id) From 09e29c3d29787d44a47d0f5e2bd6b10be84bf26f Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 8 Oct 2020 13:34:16 +0200 Subject: [PATCH 06/13] do not process tasks on entity creation but just add it's id to `modified_tasks_ftrackids` so it's processed afterwards --- pype/modules/ftrack/events/event_sync_to_avalon.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/pype/modules/ftrack/events/event_sync_to_avalon.py b/pype/modules/ftrack/events/event_sync_to_avalon.py index f2a94da56c..9df3c2838f 100644 --- a/pype/modules/ftrack/events/event_sync_to_avalon.py +++ b/pype/modules/ftrack/events/event_sync_to_avalon.py @@ -1092,15 +1092,8 @@ class SyncToAvalonEvent(BaseEvent): ) ) - # Tasks - tasks = {} - for child in ftrack_ent["children"]: - if child.entity_type.lower() != "task": - continue - self.log.debug("child:: {}".format(child)) - task_type = self._get_task_type(self.cur_project["id"], - child['entityId']) - tasks[child["name"]] = {"type": task_type} + # Add entity to modified so tasks are added at the end + self.modified_tasks_ftrackids.add(ftrack_ent["id"]) # Visual Parent vis_par = None @@ -1120,7 +1113,7 @@ class SyncToAvalonEvent(BaseEvent): "entityType": ftrack_ent.entity_type, "parents": parents, "hierarchy": hierarchy, - "tasks": tasks, + "tasks": {}, "visualParent": vis_par } } From be4a25dc60177eb552a443e05f70c7ae5b52ecec Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 8 Oct 2020 13:34:41 +0200 Subject: [PATCH 07/13] renamed `entities_query_by_parent_id` to `task_entities_query_by_parent_id` and modified query --- pype/modules/ftrack/events/event_sync_to_avalon.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pype/modules/ftrack/events/event_sync_to_avalon.py b/pype/modules/ftrack/events/event_sync_to_avalon.py index 9df3c2838f..4bfb648196 100644 --- a/pype/modules/ftrack/events/event_sync_to_avalon.py +++ b/pype/modules/ftrack/events/event_sync_to_avalon.py @@ -42,8 +42,8 @@ class SyncToAvalonEvent(BaseEvent): ) # useful for getting all tasks for asset - entities_query_by_parent_id = ( - "select id, name, parent_id, link, custom_attributes from TypedContext" + task_entities_query_by_parent_id = ( + "select id, name, parent_id, type_id from Task" " where project_id is \"{}\" and parent_id in ({})" ) entities_name_query_by_name = ( From bf04ce5fd05d80614e35d916bb10f8ddff960b3f Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 8 Oct 2020 13:35:32 +0200 Subject: [PATCH 08/13] added query for task types to not query them one by one --- pype/modules/ftrack/events/event_sync_to_avalon.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pype/modules/ftrack/events/event_sync_to_avalon.py b/pype/modules/ftrack/events/event_sync_to_avalon.py index 4bfb648196..93aeb3e20a 100644 --- a/pype/modules/ftrack/events/event_sync_to_avalon.py +++ b/pype/modules/ftrack/events/event_sync_to_avalon.py @@ -46,6 +46,9 @@ class SyncToAvalonEvent(BaseEvent): "select id, name, parent_id, type_id from Task" " where project_id is \"{}\" and parent_id in ({})" ) + task_types_query = ( + "select id, name from Type" + ) entities_name_query_by_name = ( "select id, name from TypedContext" " where project_id is \"{}\" and name in ({})" @@ -2192,6 +2195,12 @@ class SyncToAvalonEvent(BaseEvent): ftrack_mongo_mapping_found = {} not_found_ids = [] tasks_per_ftrack_id = {} + # Query all task types at once + task_types = self.process_session.query(self.task_types_query).all() + task_types_by_id = { + task_type["id"]: task_type + for task_type in task_types + } # prepare all tasks per parentId, eg. Avalon asset record for entity in entities: From e2262ce929b049febed5ff67e3c0cdc1952b0382 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 8 Oct 2020 13:36:04 +0200 Subject: [PATCH 09/13] make sure all entities with changed tasks are processed even if they dont have any task left --- pype/modules/ftrack/events/event_sync_to_avalon.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pype/modules/ftrack/events/event_sync_to_avalon.py b/pype/modules/ftrack/events/event_sync_to_avalon.py index 93aeb3e20a..5395fe756c 100644 --- a/pype/modules/ftrack/events/event_sync_to_avalon.py +++ b/pype/modules/ftrack/events/event_sync_to_avalon.py @@ -2194,7 +2194,12 @@ class SyncToAvalonEvent(BaseEvent): ftrack_mongo_mapping_found = {} not_found_ids = [] - tasks_per_ftrack_id = {} + # Make sure all parents have updated tasks, as they may not have any + tasks_per_ftrack_id = { + ftrack_id: {} + for ftrack_id in self.modified_tasks_ftrackids + } + # Query all task types at once task_types = self.process_session.query(self.task_types_query).all() task_types_by_id = { From 84b1ee867a0ca8f859b63d043325678b3fade3f2 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 8 Oct 2020 13:36:23 +0200 Subject: [PATCH 10/13] used task query for getting task entities --- pype/modules/ftrack/events/event_sync_to_avalon.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pype/modules/ftrack/events/event_sync_to_avalon.py b/pype/modules/ftrack/events/event_sync_to_avalon.py index 5395fe756c..a77f2c88fa 100644 --- a/pype/modules/ftrack/events/event_sync_to_avalon.py +++ b/pype/modules/ftrack/events/event_sync_to_avalon.py @@ -2188,9 +2188,16 @@ class SyncToAvalonEvent(BaseEvent): ) if not self.modified_tasks_ftrackids: return - entities = self._get_entities_for_ftrack_ids( - self.cur_project["id"], - self.modified_tasks_ftrackids) + + joined_ids = ", ".join([ + "\"{}\"".format(ftrack_id) + for ftrack_id in self.modified_tasks_ftrackids + ]) + task_entities = self.process_session.query( + self.task_entities_query_by_parent_id.format( + self.cur_project["id"], joined_ids + ) + ).all() ftrack_mongo_mapping_found = {} not_found_ids = [] From 5a21369cc1de9ecfb68e70e8f93076d719d2bf4a Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 8 Oct 2020 13:36:40 +0200 Subject: [PATCH 11/13] modified for loop in task update to use new changes --- .../ftrack/events/event_sync_to_avalon.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/pype/modules/ftrack/events/event_sync_to_avalon.py b/pype/modules/ftrack/events/event_sync_to_avalon.py index a77f2c88fa..6baf8e2cd8 100644 --- a/pype/modules/ftrack/events/event_sync_to_avalon.py +++ b/pype/modules/ftrack/events/event_sync_to_avalon.py @@ -2215,22 +2215,23 @@ class SyncToAvalonEvent(BaseEvent): } # prepare all tasks per parentId, eg. Avalon asset record - for entity in entities: - ftrack_id = entity["parent_id"] + for task_entity in task_entities: + task_type = task_types_by_id[task_entity["type_id"]] + ftrack_id = task_entity["parent_id"] if ftrack_id not in tasks_per_ftrack_id: tasks_per_ftrack_id[ftrack_id] = {} passed_regex = avalon_sync.check_regex( - entity["name"], "task", - schema_patterns=self.regex_schemas - ) + task_entity["name"], "task", + schema_patterns=self.regex_schemas + ) if not passed_regex: - entity_id = entity["id"] - self.regex_failed.append(entity_id) + self.regex_failed.append(task_entity["id"]) continue - task = {"type": entity["type"]["name"]} - tasks_per_ftrack_id[ftrack_id][entity["name"]] = task + tasks_per_ftrack_id[ftrack_id][task_entity["name"]] = { + "type": task_type["name"] + } # find avalon entity by parentId # should be there as create was run first From 7f4d33ee573a1df48576651037bb3119c75ab8cf Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 8 Oct 2020 13:36:57 +0200 Subject: [PATCH 12/13] removed unused methods --- .../ftrack/events/event_sync_to_avalon.py | 44 ------------------- 1 file changed, 44 deletions(-) diff --git a/pype/modules/ftrack/events/event_sync_to_avalon.py b/pype/modules/ftrack/events/event_sync_to_avalon.py index 6baf8e2cd8..0c76cb452e 100644 --- a/pype/modules/ftrack/events/event_sync_to_avalon.py +++ b/pype/modules/ftrack/events/event_sync_to_avalon.py @@ -2512,50 +2512,6 @@ class SyncToAvalonEvent(BaseEvent): return mongo_id_configuration_id - def _get_task_type(self, project_id, entityId): - """ - Returns task type ('Props', 'Art') from Task 'entityId'. - Args: - project_id (string): - entityId (string): entityId of Task - - Returns: - (string) - None if Task not found - """ - task_type = None - entity = self.process_session.query( - self.entities_query_by_id.format( - project_id, entityId - ) - ).first() - if entity: - task_type = entity["type"]["name"] - return task_type - - def _get_entities_for_ftrack_ids(self, ft_project_id, ftrack_ids): - """ - Query Ftrack API and return all entities for particular - 'ft_project' and their parent_id in 'ftrack_ids'. - It is much faster to run this once for multiple ids than run it - for each separately. - Used mainly for collecting task information - Args: - ft_project_id (string): - ftrack_ids (list): of strings - - Returns: - (list) of Ftrack entities - """ - ftrack_entities = [] - if ftrack_ids: - joined_ids = ", ".join(["\"{}\"".format(id) for id in ftrack_ids]) - ftrack_entities = self.process_session.query( - self.entities_query_by_parent_id.format(ft_project_id, - joined_ids) - ).all() - - return ftrack_entities - def register(session, plugins_presets): '''Register plugin. Called when used as an plugin.''' From 9225c3553de6c673cf2a5351fd4b9feaccd91a64 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 8 Oct 2020 13:37:05 +0200 Subject: [PATCH 13/13] formatting changes --- .../ftrack/events/event_sync_to_avalon.py | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/pype/modules/ftrack/events/event_sync_to_avalon.py b/pype/modules/ftrack/events/event_sync_to_avalon.py index 0c76cb452e..251d87331e 100644 --- a/pype/modules/ftrack/events/event_sync_to_avalon.py +++ b/pype/modules/ftrack/events/event_sync_to_avalon.py @@ -739,13 +739,13 @@ class SyncToAvalonEvent(BaseEvent): time_cleanup = time_7 - time_6 time_task_updates = time_8 - time_7 time_total = time_8 - time_1 - self.log.debug( - "Process time: {:.2f} <{:.2f}, {:.2f}, {:.2f}, ".format( - time_total, time_removed, time_renamed, time_added) + - "{:.2f}, {:.2f}, {:.2f}, {:.2f}>".format( - time_moved, time_updated, time_cleanup, time_task_updates - ) - ) + self.log.debug(( + "Process time: {:.2f} <{:.2f}, {:.2f}, {:.2f}, " + "{:.2f}, {:.2f}, {:.2f}, {:.2f}>" + ).format( + time_total, time_removed, time_renamed, time_added, + time_moved, time_updated, time_cleanup, time_task_updates + )) except Exception: msg = "An error has happened during synchronization" @@ -2242,8 +2242,10 @@ class SyncToAvalonEvent(BaseEvent): continue ftrack_mongo_mapping_found[ftrack_id] = avalon_entity["_id"] - self._update_avalon_tasks(ftrack_mongo_mapping_found, - tasks_per_ftrack_id) + self._update_avalon_tasks( + ftrack_mongo_mapping_found, + tasks_per_ftrack_id + ) def update_entities(self): """ @@ -2441,8 +2443,9 @@ class SyncToAvalonEvent(BaseEvent): ) return True - def _update_avalon_tasks(self, ftrack_mongo_mapping_found, - tasks_per_ftrack_id): + def _update_avalon_tasks( + self, ftrack_mongo_mapping_found, tasks_per_ftrack_id + ): """ Prepare new "tasks" content for existing records in Avalon. Args: @@ -2460,10 +2463,9 @@ class SyncToAvalonEvent(BaseEvent): change_data = {"$set": {}} change_data["$set"]["data.tasks"] = tasks_per_ftrack_id[ftrack_id] mongo_changes_bulk.append(UpdateOne(filter, change_data)) - if not mongo_changes_bulk: - return - self.dbcon.bulk_write(mongo_changes_bulk) + if mongo_changes_bulk: + self.dbcon.bulk_write(mongo_changes_bulk) def _mongo_id_configuration( self,