From 4c6e5fefb415eab87ee52fd3b0bceb416f0643bf Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 20 Aug 2020 16:05:03 +0200 Subject: [PATCH 01/13] implemented action for pushing frameStart and frameEnd values to task specific custom attributes --- .../action_push_frame_values_to_task.py | 245 ++++++++++++++++++ 1 file changed, 245 insertions(+) create mode 100644 pype/modules/ftrack/actions/action_push_frame_values_to_task.py diff --git a/pype/modules/ftrack/actions/action_push_frame_values_to_task.py b/pype/modules/ftrack/actions/action_push_frame_values_to_task.py new file mode 100644 index 0000000000..3037695452 --- /dev/null +++ b/pype/modules/ftrack/actions/action_push_frame_values_to_task.py @@ -0,0 +1,245 @@ +import json +import collections +import ftrack_api +from pype.modules.ftrack.lib import BaseAction, statics_icon + + +class PushFrameValuesToTaskAction(BaseAction): + """Action for testing purpose or as base for new actions.""" + + identifier = "admin.push_frame_values_to_task" + label = "Pype Admin" + variant = "- Push Frame values to Task" + role_list = ["Pypeclub", "Administrator", "Project Manager"] + icon = statics_icon("ftrack", "action_icons", "PypeAdmin.svg") + + entities_query = ( + "select id, name, parent_id, link" + " from TypedContext where project_id is \"{}\"" + ) + cust_attrs_query = ( + "select id, key, object_type_id, is_hierarchical, default" + " from CustomAttributeConfiguration" + " where key in ({})" + ) + cust_attr_value_query = ( + "select value, entity_id from CustomAttributeValue" + " where entity_id in ({}) and configuration_id in ({})" + ) + custom_attribute_keys = ["frameStart", "frameEnd"] + + def discover(self, session, entities, event): + return True + + def launch(self, session, entities, event): + task_attrs_by_key, hier_attrs = self.frame_attributes(session) + missing_keys = [ + key + for key in self.custom_attribute_keys + if key not in task_attrs_by_key + ] + if missing_keys: + if len(missing_keys) == 1: + sub_msg = " \"{}\"".format(missing_keys[0]) + else: + sub_msg = "s {}".format(", ".join([ + "\"{}\"".format(key) + for key in missing_keys + ])) + + msg = "Missing Task's custom attribute{}.".format(sub_msg) + self.log.warning(msg) + return { + "success": False, + "message": msg + } + + self.log.debug("{}: Creating job".format(self.label)) + + user_entity = session.query( + "User where id is {}".format(event["source"]["user"]["id"]) + ).one() + job = session.create("Job", { + "user": user_entity, + "status": "running", + "data": json.dumps({ + "description": "Propagation of Frame attribute values to task." + }) + }) + session.commit() + + try: + project_entity = self.get_project_from_entity(entities[0]) + result = self.propagate_values( + session, + tuple(task_attrs_by_key.values()), + hier_attrs, + project_entity + ) + job["status"] = "done" + session.commit() + + return result + + except Exception: + session.rollback() + job["status"] = "failed" + session.commit() + + msg = "Pushing Custom attribute values to task Failed" + self.log.warning(msg, exc_info=True) + return { + "success": False, + "message": msg + } + + finally: + if job["status"] == "running": + job["status"] = "failed" + session.commit() + + def frame_attributes(self, session): + task_object_type = session.query( + "ObjectType where name is \"Task\"" + ).one() + + attr_names = self.custom_attribute_keys + if isinstance(attr_names, str): + attr_names = [attr_names] + + joined_keys = ",".join([ + "\"{}\"".format(attr_name) + for attr_name in attr_names + ]) + + attribute_entities = session.query( + self.cust_attrs_query.format(joined_keys) + ).all() + + hier_attrs = [] + task_attrs = {} + for attr in attribute_entities: + attr_key = attr["key"] + if attr["is_hierarchical"]: + hier_attrs.append(attr) + elif attr["object_type_id"] == task_object_type["id"]: + task_attrs[attr_key] = attr + return task_attrs, hier_attrs + + def join_keys(self, items): + return ",".join(["\"{}\"".format(item) for item in items]) + + def propagate_values( + self, session, task_attrs, hier_attrs, project_entity + ): + self.log.debug("Querying project's entities \"{}\".".format( + project_entity["full_name"] + )) + entities = session.query( + self.entities_query.format(project_entity["id"]) + ).all() + + self.log.debug("Filtering Task entities.") + task_entities_by_parent_id = collections.defaultdict(list) + for entity in entities: + if entity.entity_type.lower() == "task": + task_entities_by_parent_id[entity["parent_id"]].append(entity) + + self.log.debug("Getting Custom attribute values from tasks' parents.") + hier_values_by_entity_id = self.get_hier_values( + session, + hier_attrs, + list(task_entities_by_parent_id.keys()) + ) + + self.log.debug("Setting parents' values to task.") + self.set_task_attr_values( + session, + task_entities_by_parent_id, + hier_values_by_entity_id, + task_attrs + ) + + return True + + def get_hier_values(self, session, hier_attrs, focus_entity_ids): + joined_entity_ids = self.join_keys(focus_entity_ids) + hier_attr_ids = self.join_keys( + tuple(hier_attr["id"] for hier_attr in hier_attrs) + ) + hier_attrs_key_by_id = { + hier_attr["id"]: hier_attr["key"] + for hier_attr in hier_attrs + } + call_expr = [{ + "action": "query", + "expression": self.cust_attr_value_query.format( + joined_entity_ids, hier_attr_ids + ) + }] + if hasattr(session, "call"): + [values] = session.call(call_expr) + else: + [values] = session._call(call_expr) + + values_per_entity_id = {} + for item in values["data"]: + entity_id = item["entity_id"] + key = hier_attrs_key_by_id[item["configuration_id"]] + + if entity_id not in values_per_entity_id: + values_per_entity_id[entity_id] = {} + value = item["value"] + if value is not None: + values_per_entity_id[entity_id][key] = value + + output = {} + for entity_id in focus_entity_ids: + value = values_per_entity_id.get(entity_id) + if value: + output[entity_id] = value + + return output + + def set_task_attr_values( + self, + session, + task_entities_by_parent_id, + hier_values_by_entity_id, + task_attrs + ): + task_attr_ids_by_key = { + attr["key"]: attr["id"] + for attr in task_attrs + } + + total_parents = len(hier_values_by_entity_id) + idx = 1 + for parent_id, values in hier_values_by_entity_id.items(): + self.log.info(( + "[{}/{}] {} Processing values to children. Values: {}" + ).format(idx, total_parents, parent_id, values)) + + task_entities = task_entities_by_parent_id[parent_id] + for key, value in values.items(): + for task_entity in task_entities: + _entity_key = collections.OrderedDict({ + "configuration_id": task_attr_ids_by_key[key], + "entity_id": task_entity["id"] + }) + + session.recorded_operations.push( + ftrack_api.operation.UpdateEntityOperation( + "ContextCustomAttributeValue", + _entity_key, + "value", + ftrack_api.symbol.NOT_SET, + value + ) + ) + session.commit() + idx += 1 + + +def register(session, plugins_presets={}): + PushFrameValuesToTask(session, plugins_presets).register() From 1431930e6af89df656cced3572598907d13525ad Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 20 Aug 2020 16:05:13 +0200 Subject: [PATCH 02/13] implemented event handler for pushing frameStart and frameEnd values to task specific custom attributes --- .../events/event_push_frame_values_to_task.py | 148 ++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 pype/modules/ftrack/events/event_push_frame_values_to_task.py diff --git a/pype/modules/ftrack/events/event_push_frame_values_to_task.py b/pype/modules/ftrack/events/event_push_frame_values_to_task.py new file mode 100644 index 0000000000..dd5c5911ec --- /dev/null +++ b/pype/modules/ftrack/events/event_push_frame_values_to_task.py @@ -0,0 +1,148 @@ +import collections +import ftrack_api +from pype.modules.ftrack import BaseEvent + + +class PushFrameValuesToTaskEvent(BaseEvent): + """Action for testing purpose or as base for new actions.""" + cust_attrs_query = ( + "select id, key, object_type_id, is_hierarchical, default" + " from CustomAttributeConfiguration" + " where key in ({}) and object_type_id = {}" + ) + + # Ignore event handler by default + ignore_me = True + + interest_attributes = ["frameStart", "frameEnd"] + _cached_task_object_id = None + + @classmethod + def task_object_id(cls, session): + if cls._cached_task_object_id is None: + task_object_type = session.query( + "ObjectType where name is \"Task\"" + ).one() + cls._cached_task_object_id = task_object_type["id"] + return cls._cached_task_object_id + + def extract_interesting_data(self, session, event): + # Filter if event contain relevant data + entities_info = event["data"].get("entities") + if not entities_info: + return + + interesting_data = {} + for entity_info in entities_info: + # Care only about tasks + if entity_info.get("entityType") != "task": + continue + + # Care only about changes of status + changes = entity_info.get("changes") or {} + if not changes: + continue + + # Care only about changes if specific keys + entity_changes = {} + for key in self.interest_attributes: + if key in changes: + entity_changes[key] = changes[key]["new"] + + if not entity_changes: + continue + + # Do not care about "Task" entity_type + task_object_id = self.task_object_id(session) + if entity_info.get("objectTypeId") == task_object_id: + continue + + interesting_data[entity_info["entityId"]] = entity_changes + return interesting_data + + def join_keys(self, keys): + return ",".join(["\"{}\"".format(key) for key in keys]) + + def get_task_entities(self, session, entities_info): + return session.query( + "Task where parent_id in ({})".format( + self.join_keys(entities_info.keys()) + ) + ).all() + + def task_attrs(self, session): + return session.query(self.cust_attrs_query.format( + self.join_keys(self.interest_attributes), + self.task_object_id(session) + )).all() + + def launch(self, session, event): + interesting_data = self.extract_interesting_data(session, event) + if not interesting_data: + return + + task_entities = self.get_task_entities(session, interesting_data) + if not task_entities: + return + + task_attrs = self.task_attrs(session) + if not task_attrs: + self.log.warning(( + "There is not created Custom Attributes {}" + " for \"Task\" entity type." + ).format(self.join_keys(self.interest_attributes))) + return + + task_attr_id_by_key = { + attr["key"]: attr["id"] + for attr in task_attrs + } + task_entities_by_parent_id = collections.defaultdict(list) + for task_entity in task_entities: + task_entities_by_parent_id[task_entity["parent_id"]].append( + task_entity + ) + + for parent_id, values in interesting_data.items(): + task_entities = task_entities_by_parent_id[parent_id] + for key, value in values.items(): + changed_ids = [] + for task_entity in task_entities: + task_id = task_entity["id"] + changed_ids.append(task_id) + + entity_key = collections.OrderedDict({ + "configuration_id": task_attr_id_by_key[key], + "entity_id": task_id + }) + if value is None: + op = ftrack_api.operation.DeleteEntityOperation( + "CustomAttributeValue", + entity_key + ) + else: + op = ftrack_api.operation.UpdateEntityOperation( + "ContextCustomAttributeValue", + entity_key, + "value", + ftrack_api.symbol.NOT_SET, + value + ) + + session.recorded_operations.push(op) + self.log.info(( + "Changing Custom Attribute \"{}\" to value" + " \"{}\" on entities: {}" + ).format(key, value, self.join_keys(changed_ids))) + try: + session.commit() + except Exception: + session.rollback() + self.log.warning( + "Changing of values failed.", + exc_info=True + ) + + +def register(session, plugins_presets): + PushFrameValuesToTaskEvent(session, plugins_presets).register() From 516fafbfec822e8b5f2957bdd108f157412a5963 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 20 Aug 2020 16:39:16 +0200 Subject: [PATCH 03/13] moved action to server --- .../{actions => events}/action_push_frame_values_to_task.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pype/modules/ftrack/{actions => events}/action_push_frame_values_to_task.py (100%) diff --git a/pype/modules/ftrack/actions/action_push_frame_values_to_task.py b/pype/modules/ftrack/events/action_push_frame_values_to_task.py similarity index 100% rename from pype/modules/ftrack/actions/action_push_frame_values_to_task.py rename to pype/modules/ftrack/events/action_push_frame_values_to_task.py From 8ae527a154c24d0b0e5634546d759f421f0c0426 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 20 Aug 2020 16:39:47 +0200 Subject: [PATCH 04/13] action converted to server action --- .../action_push_frame_values_to_task.py | 64 ++++++++++++++----- 1 file changed, 49 insertions(+), 15 deletions(-) diff --git a/pype/modules/ftrack/events/action_push_frame_values_to_task.py b/pype/modules/ftrack/events/action_push_frame_values_to_task.py index 3037695452..bd036411ac 100644 --- a/pype/modules/ftrack/events/action_push_frame_values_to_task.py +++ b/pype/modules/ftrack/events/action_push_frame_values_to_task.py @@ -1,7 +1,7 @@ import json import collections import ftrack_api -from pype.modules.ftrack.lib import BaseAction, statics_icon +from pype.modules.ftrack.lib import BaseAction class PushFrameValuesToTaskAction(BaseAction): @@ -10,8 +10,6 @@ class PushFrameValuesToTaskAction(BaseAction): identifier = "admin.push_frame_values_to_task" label = "Pype Admin" variant = "- Push Frame values to Task" - role_list = ["Pypeclub", "Administrator", "Project Manager"] - icon = statics_icon("ftrack", "action_icons", "PypeAdmin.svg") entities_query = ( "select id, name, parent_id, link" @@ -26,12 +24,56 @@ class PushFrameValuesToTaskAction(BaseAction): "select value, entity_id from CustomAttributeValue" " where entity_id in ({}) and configuration_id in ({})" ) - custom_attribute_keys = ["frameStart", "frameEnd"] + custom_attribute_keys = {"frameStart", "frameEnd"} + discover_role_list = {"Pypeclub", "Administrator", "Project Manager"} + + def register(self): + modified_role_names = set() + for role_name in self.discover_role_list: + modified_role_names.add(role_name.lower()) + self.discover_role_list = modified_role_names + + self.session.event_hub.subscribe( + "topic=ftrack.action.discover", + self._discover, + priority=self.priority + ) + + launch_subscription = ( + "topic=ftrack.action.launch and data.actionIdentifier={0}" + ).format(self.identifier) + self.session.event_hub.subscribe(launch_subscription, self._launch) def discover(self, session, entities, event): - return True + """ Validation """ + # Check if selection is valid + valid_selection = False + for ent in event["data"]["selection"]: + # Ignore entities that are not tasks or projects + if ent["entityType"].lower() in ["show", "task"]: + valid_selection = True + break + + if not valid_selection: + return False + + # Get user and check his roles + user_id = event.get("source", {}).get("user", {}).get("id") + if not user_id: + return False + + user = session.query("User where id is \"{}\"".format(user_id)).first() + if not user: + return False + + for role in user["user_security_roles"]: + lowered_role = role["security_role"]["name"].lower() + if lowered_role in self.discover_role_list: + return True + return False def launch(self, session, entities, event): + # TODO this can be threaded task_attrs_by_key, hier_attrs = self.frame_attributes(session) missing_keys = [ key @@ -103,15 +145,7 @@ class PushFrameValuesToTaskAction(BaseAction): "ObjectType where name is \"Task\"" ).one() - attr_names = self.custom_attribute_keys - if isinstance(attr_names, str): - attr_names = [attr_names] - - joined_keys = ",".join([ - "\"{}\"".format(attr_name) - for attr_name in attr_names - ]) - + joined_keys = self.join_keys(self.custom_attribute_keys) attribute_entities = session.query( self.cust_attrs_query.format(joined_keys) ).all() @@ -242,4 +276,4 @@ class PushFrameValuesToTaskAction(BaseAction): def register(session, plugins_presets={}): - PushFrameValuesToTask(session, plugins_presets).register() + PushFrameValuesToTaskAction(session, plugins_presets).register() From a37da37bd15c6e5aca9c37ea56a57eb163359f11 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 20 Aug 2020 16:42:56 +0200 Subject: [PATCH 05/13] commit all changes at once --- .../modules/ftrack/events/action_push_frame_values_to_task.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pype/modules/ftrack/events/action_push_frame_values_to_task.py b/pype/modules/ftrack/events/action_push_frame_values_to_task.py index bd036411ac..4f0c7ffeb7 100644 --- a/pype/modules/ftrack/events/action_push_frame_values_to_task.py +++ b/pype/modules/ftrack/events/action_push_frame_values_to_task.py @@ -253,6 +253,7 @@ class PushFrameValuesToTaskAction(BaseAction): self.log.info(( "[{}/{}] {} Processing values to children. Values: {}" ).format(idx, total_parents, parent_id, values)) + idx += 1 task_entities = task_entities_by_parent_id[parent_id] for key, value in values.items(): @@ -271,8 +272,7 @@ class PushFrameValuesToTaskAction(BaseAction): value ) ) - session.commit() - idx += 1 + session.commit() def register(session, plugins_presets={}): From 97b42d8703150038feb6833832c9da0d39df40ce Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 20 Aug 2020 16:46:35 +0200 Subject: [PATCH 06/13] ignore action by default --- pype/modules/ftrack/events/action_push_frame_values_to_task.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pype/modules/ftrack/events/action_push_frame_values_to_task.py b/pype/modules/ftrack/events/action_push_frame_values_to_task.py index 4f0c7ffeb7..5b7da8bebb 100644 --- a/pype/modules/ftrack/events/action_push_frame_values_to_task.py +++ b/pype/modules/ftrack/events/action_push_frame_values_to_task.py @@ -7,6 +7,9 @@ from pype.modules.ftrack.lib import BaseAction class PushFrameValuesToTaskAction(BaseAction): """Action for testing purpose or as base for new actions.""" + # Ignore event handler by default + ignore_me = True + identifier = "admin.push_frame_values_to_task" label = "Pype Admin" variant = "- Push Frame values to Task" From 07b34dec926f0135a03327d4c52aa2d2837e5199 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 20 Aug 2020 17:45:59 +0200 Subject: [PATCH 07/13] show only on project --- pype/modules/ftrack/events/action_push_frame_values_to_task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/modules/ftrack/events/action_push_frame_values_to_task.py b/pype/modules/ftrack/events/action_push_frame_values_to_task.py index 5b7da8bebb..e6276d84ac 100644 --- a/pype/modules/ftrack/events/action_push_frame_values_to_task.py +++ b/pype/modules/ftrack/events/action_push_frame_values_to_task.py @@ -53,7 +53,7 @@ class PushFrameValuesToTaskAction(BaseAction): valid_selection = False for ent in event["data"]["selection"]: # Ignore entities that are not tasks or projects - if ent["entityType"].lower() in ["show", "task"]: + if ent["entityType"].lower() == "show": valid_selection = True break From fb6de46cd6c186934cd878c6e055dadf86c89d83 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 20 Aug 2020 19:05:56 +0200 Subject: [PATCH 08/13] pushing is also pushing to item itself --- .../action_push_frame_values_to_task.py | 279 ++++++++++++++---- 1 file changed, 221 insertions(+), 58 deletions(-) diff --git a/pype/modules/ftrack/events/action_push_frame_values_to_task.py b/pype/modules/ftrack/events/action_push_frame_values_to_task.py index e6276d84ac..d88f4a1016 100644 --- a/pype/modules/ftrack/events/action_push_frame_values_to_task.py +++ b/pype/modules/ftrack/events/action_push_frame_values_to_task.py @@ -15,8 +15,8 @@ class PushFrameValuesToTaskAction(BaseAction): variant = "- Push Frame values to Task" entities_query = ( - "select id, name, parent_id, link" - " from TypedContext where project_id is \"{}\"" + "select id, name, parent_id, link from TypedContext" + " where project_id is \"{}\" and object_type_id in ({})" ) cust_attrs_query = ( "select id, key, object_type_id, is_hierarchical, default" @@ -27,7 +27,13 @@ class PushFrameValuesToTaskAction(BaseAction): "select value, entity_id from CustomAttributeValue" " where entity_id in ({}) and configuration_id in ({})" ) - custom_attribute_keys = {"frameStart", "frameEnd"} + + pushing_entity_types = {"Shot"} + hierarchical_custom_attribute_keys = {"frameStart", "frameEnd"} + custom_attribute_mapping = { + "frameStart": "fstart", + "frameEnd": "fend" + } discover_role_list = {"Pypeclub", "Administrator", "Project Manager"} def register(self): @@ -76,29 +82,6 @@ class PushFrameValuesToTaskAction(BaseAction): return False def launch(self, session, entities, event): - # TODO this can be threaded - task_attrs_by_key, hier_attrs = self.frame_attributes(session) - missing_keys = [ - key - for key in self.custom_attribute_keys - if key not in task_attrs_by_key - ] - if missing_keys: - if len(missing_keys) == 1: - sub_msg = " \"{}\"".format(missing_keys[0]) - else: - sub_msg = "s {}".format(", ".join([ - "\"{}\"".format(key) - for key in missing_keys - ])) - - msg = "Missing Task's custom attribute{}.".format(sub_msg) - self.log.warning(msg) - return { - "success": False, - "message": msg - } - self.log.debug("{}: Creating job".format(self.label)) user_entity = session.query( @@ -115,12 +98,7 @@ class PushFrameValuesToTaskAction(BaseAction): try: project_entity = self.get_project_from_entity(entities[0]) - result = self.propagate_values( - session, - tuple(task_attrs_by_key.values()), - hier_attrs, - project_entity - ) + result = self.propagate_values(session, project_entity, event) job["status"] = "done" session.commit() @@ -143,12 +121,20 @@ class PushFrameValuesToTaskAction(BaseAction): job["status"] = "failed" session.commit() - def frame_attributes(self, session): + def task_attributes(self, session): task_object_type = session.query( "ObjectType where name is \"Task\"" ).one() - joined_keys = self.join_keys(self.custom_attribute_keys) + hier_attr_names = list( + self.custom_attribute_mapping.keys() + ) + entity_type_specific_names = list( + self.custom_attribute_mapping.values() + ) + joined_keys = self.join_keys( + hier_attr_names + entity_type_specific_names + ) attribute_entities = session.query( self.cust_attrs_query.format(joined_keys) ).all() @@ -158,47 +144,139 @@ class PushFrameValuesToTaskAction(BaseAction): for attr in attribute_entities: attr_key = attr["key"] if attr["is_hierarchical"]: - hier_attrs.append(attr) + if attr_key in hier_attr_names: + hier_attrs.append(attr) elif attr["object_type_id"] == task_object_type["id"]: - task_attrs[attr_key] = attr + if attr_key in entity_type_specific_names: + task_attrs[attr_key] = attr["id"] return task_attrs, hier_attrs def join_keys(self, items): return ",".join(["\"{}\"".format(item) for item in items]) - def propagate_values( - self, session, task_attrs, hier_attrs, project_entity - ): + def propagate_values(self, session, project_entity, event): self.log.debug("Querying project's entities \"{}\".".format( project_entity["full_name"] )) - entities = session.query( - self.entities_query.format(project_entity["id"]) - ).all() + pushing_entity_types = tuple( + ent_type.lower() + for ent_type in self.pushing_entity_types + ) + destination_object_types = [] + all_object_types = session.query("ObjectType").all() + for object_type in all_object_types: + lowered_name = object_type["name"].lower() + if ( + lowered_name == "task" + or lowered_name in pushing_entity_types + ): + destination_object_types.append(object_type) + + destination_object_type_ids = tuple( + obj_type["id"] + for obj_type in destination_object_types + ) + entities = session.query(self.entities_query.format( + project_entity["id"], + self.join_keys(destination_object_type_ids) + )).all() + + entities_by_id = { + entity["id"]: entity + for entity in entities + } self.log.debug("Filtering Task entities.") task_entities_by_parent_id = collections.defaultdict(list) + non_task_entities = [] + non_task_entity_ids = [] for entity in entities: - if entity.entity_type.lower() == "task": - task_entities_by_parent_id[entity["parent_id"]].append(entity) + if entity.entity_type.lower() != "task": + non_task_entities.append(entity) + non_task_entity_ids.append(entity["id"]) + continue + + parent_id = entity["parent_id"] + if parent_id in entities_by_id: + task_entities_by_parent_id[parent_id].append(entity) + + task_attr_id_by_keys, hier_attrs = self.task_attributes(session) self.log.debug("Getting Custom attribute values from tasks' parents.") hier_values_by_entity_id = self.get_hier_values( session, hier_attrs, - list(task_entities_by_parent_id.keys()) + non_task_entity_ids ) self.log.debug("Setting parents' values to task.") - self.set_task_attr_values( + task_missing_keys = self.set_task_attr_values( session, task_entities_by_parent_id, hier_values_by_entity_id, - task_attrs + task_attr_id_by_keys ) + self.log.debug("Setting values to entities themselves.") + missing_keys_by_object_name = self.push_values_to_entities( + session, + non_task_entities, + hier_values_by_entity_id + ) + if task_missing_keys: + missing_keys_by_object_name["Task"] = task_missing_keys + if missing_keys_by_object_name: + self.report(missing_keys_by_object_name, event) return True + def report(self, missing_keys_by_object_name, event): + splitter = {"type": "label", "value": "---"} + + title = "Push Custom Attribute values report:" + + items = [] + items.append({ + "type": "label", + "value": "# Pushing values was not complete" + }) + items.append({ + "type": "label", + "value": ( + "

It was due to missing custom" + " attribute configurations for specific entity type/s." + " These configurations are not created automatically.

" + ) + }) + + log_message_items = [] + log_message_item_template = ( + "Entity type \"{}\" does not have created Custom Attribute/s: {}" + ) + for object_name, missing_attr_names in ( + missing_keys_by_object_name.items() + ): + log_message_items.append(log_message_item_template.format( + object_name, self.join_keys(missing_attr_names) + )) + + items.append(splitter) + items.append({ + "type": "label", + "value": "## Entity type: {}".format(object_name) + }) + + items.append({ + "type": "label", + "value": "

{}

".format("
".join(missing_attr_names)) + }) + + self.log.warning(( + "Couldn't finish pushing attribute values because" + " few entity types miss Custom attribute configurations:\n{}" + ).format("\n".join(log_message_items))) + + self.show_interface(items, title, event) + def get_hier_values(self, session, hier_attrs, focus_entity_ids): joined_entity_ids = self.join_keys(focus_entity_ids) hier_attr_ids = self.join_keys( @@ -243,26 +321,28 @@ class PushFrameValuesToTaskAction(BaseAction): session, task_entities_by_parent_id, hier_values_by_entity_id, - task_attrs + task_attr_id_by_keys ): - task_attr_ids_by_key = { - attr["key"]: attr["id"] - for attr in task_attrs - } + missing_keys = set() total_parents = len(hier_values_by_entity_id) - idx = 1 + idx = 0 for parent_id, values in hier_values_by_entity_id.items(): - self.log.info(( - "[{}/{}] {} Processing values to children. Values: {}" - ).format(idx, total_parents, parent_id, values)) idx += 1 + self.log.info(( + "[{}/{}] {} Processing values to task. Values: {}" + ).format(idx, total_parents, parent_id, values)) task_entities = task_entities_by_parent_id[parent_id] - for key, value in values.items(): + for hier_key, value in values.items(): + key = self.custom_attribute_mapping[hier_key] + if key not in task_attr_id_by_keys: + missing_keys.add(key) + continue + for task_entity in task_entities: _entity_key = collections.OrderedDict({ - "configuration_id": task_attr_ids_by_key[key], + "configuration_id": task_attr_id_by_keys[key], "entity_id": task_entity["id"] }) @@ -277,6 +357,89 @@ class PushFrameValuesToTaskAction(BaseAction): ) session.commit() + return missing_keys + + def push_values_to_entities( + self, + session, + non_task_entities, + hier_values_by_entity_id + ): + object_types = session.query( + "ObjectType where name in ({})".format( + self.join_keys(self.pushing_entity_types) + ) + ).all() + object_type_names_by_id = { + object_type["id"]: object_type["name"] + for object_type in object_types + } + joined_keys = self.join_keys( + self.custom_attribute_mapping.values() + ) + attribute_entities = session.query( + self.cust_attrs_query.format(joined_keys) + ).all() + + attrs_by_obj_id = {} + for attr in attribute_entities: + if attr["is_hierarchical"]: + continue + + obj_id = attr["object_type_id"] + if obj_id not in object_type_names_by_id: + continue + + if obj_id not in attrs_by_obj_id: + attrs_by_obj_id[obj_id] = {} + + attr_key = attr["key"] + attrs_by_obj_id[obj_id][attr_key] = attr["id"] + + entities_by_obj_id = collections.defaultdict(list) + for entity in non_task_entities: + entities_by_obj_id[entity["object_type_id"]].append(entity) + + missing_keys_by_object_id = collections.defaultdict(set) + for obj_type_id, attr_keys in attrs_by_obj_id.items(): + entities = entities_by_obj_id.get(obj_type_id) + if not entities: + continue + + for entity in entities: + values = hier_values_by_entity_id.get(entity["id"]) + if not values: + continue + + for hier_key, value in values.items(): + key = self.custom_attribute_mapping[hier_key] + if key not in attr_keys: + missing_keys_by_object_id[obj_type_id].add(key) + continue + + _entity_key = collections.OrderedDict({ + "configuration_id": attr_keys[key], + "entity_id": entity["id"] + }) + + session.recorded_operations.push( + ftrack_api.operation.UpdateEntityOperation( + "ContextCustomAttributeValue", + _entity_key, + "value", + ftrack_api.symbol.NOT_SET, + value + ) + ) + session.commit() + + missing_keys_by_object_name = {} + for obj_id, missing_keys in missing_keys_by_object_id.items(): + obj_name = object_type_names_by_id[obj_id] + missing_keys_by_object_name[obj_name] = missing_keys + + return missing_keys_by_object_name + def register(session, plugins_presets={}): PushFrameValuesToTaskAction(session, plugins_presets).register() From 044434b35205f10e0d7d415d7d32ecefdbd65257 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 20 Aug 2020 19:32:36 +0200 Subject: [PATCH 09/13] event handle the same way as action --- .../events/event_push_frame_values_to_task.py | 145 ++++++++++++++---- 1 file changed, 114 insertions(+), 31 deletions(-) diff --git a/pype/modules/ftrack/events/event_push_frame_values_to_task.py b/pype/modules/ftrack/events/event_push_frame_values_to_task.py index dd5c5911ec..dd24110c1b 100644 --- a/pype/modules/ftrack/events/event_push_frame_values_to_task.py +++ b/pype/modules/ftrack/events/event_push_frame_values_to_task.py @@ -4,18 +4,27 @@ from pype.modules.ftrack import BaseEvent class PushFrameValuesToTaskEvent(BaseEvent): - """Action for testing purpose or as base for new actions.""" - cust_attrs_query = ( - "select id, key, object_type_id, is_hierarchical, default" - " from CustomAttributeConfiguration" - " where key in ({}) and object_type_id = {}" - ) - # Ignore event handler by default ignore_me = True - interest_attributes = ["frameStart", "frameEnd"] + cust_attrs_query = ( + "select id, key, object_type_id, is_hierarchical, default" + " from CustomAttributeConfiguration" + " where key in ({}) and object_type_id in ({})" + ) + + interest_entity_types = {"Shot"} + interest_attributes = {"frameStart", "frameEnd"} + interest_attr_mapping = { + "frameStart": "fstart", + "frameEnd": "fend" + } _cached_task_object_id = None + _cached_interest_object_ids = None + + @staticmethod + def join_keys(keys): + return ",".join(["\"{}\"".format(key) for key in keys]) @classmethod def task_object_id(cls, session): @@ -26,6 +35,20 @@ class PushFrameValuesToTaskEvent(BaseEvent): cls._cached_task_object_id = task_object_type["id"] return cls._cached_task_object_id + @classmethod + def interest_object_ids(cls, session): + if cls._cached_interest_object_ids is None: + object_types = session.query( + "ObjectType where name in ({})".format( + cls.join_keys(cls.interest_entity_types) + ) + ).all() + cls._cached_interest_object_ids = tuple( + object_type["id"] + for object_type in object_types + ) + return cls._cached_interest_object_ids + def extract_interesting_data(self, session, event): # Filter if event contain relevant data entities_info = event["data"].get("entities") @@ -60,60 +83,107 @@ class PushFrameValuesToTaskEvent(BaseEvent): interesting_data[entity_info["entityId"]] = entity_changes return interesting_data - def join_keys(self, keys): - return ",".join(["\"{}\"".format(key) for key in keys]) - - def get_task_entities(self, session, entities_info): - return session.query( - "Task where parent_id in ({})".format( - self.join_keys(entities_info.keys()) + def get_entities(self, session, interesting_data): + entities = session.query( + "TypedContext where id in ({})".format( + self.join_keys(interesting_data.keys()) ) ).all() - def task_attrs(self, session): - return session.query(self.cust_attrs_query.format( - self.join_keys(self.interest_attributes), - self.task_object_id(session) + output = [] + interest_object_ids = self.interest_object_ids(session) + for entity in entities: + if entity["object_type_id"] in interest_object_ids: + output.append(entity) + return output + + def get_task_entities(self, session, interesting_data): + return session.query( + "Task where parent_id in ({})".format( + self.join_keys(interesting_data.keys()) + ) + ).all() + + def attrs_configurations(self, session): + object_ids = list(self.interest_object_ids(session)) + object_ids.append(self.task_object_id(session)) + + attrs = session.query(self.cust_attrs_query.format( + self.join_keys(self.interest_attr_mapping.values()), + self.join_keys(object_ids) )).all() + output = {} + for attr in attrs: + obj_id = attr["object_type_id"] + if obj_id not in output: + output[obj_id] = {} + output[obj_id][attr["key"]] = attr["id"] + return output + def launch(self, session, event): interesting_data = self.extract_interesting_data(session, event) if not interesting_data: return + entities = self.get_entities(session, interesting_data) + if not entities: + return + + entities_by_id = { + entity["id"]: entity + for entity in entities + } + for entity_id in tuple(interesting_data.keys()): + if entity_id not in entities_by_id: + interesting_data.pop(entity_id) + task_entities = self.get_task_entities(session, interesting_data) if not task_entities: return - task_attrs = self.task_attrs(session) - if not task_attrs: + attrs_by_obj_id = self.attrs_configurations(session) + if not attrs_by_obj_id: self.log.warning(( "There is not created Custom Attributes {}" " for \"Task\" entity type." ).format(self.join_keys(self.interest_attributes))) return - task_attr_id_by_key = { - attr["key"]: attr["id"] - for attr in task_attrs - } task_entities_by_parent_id = collections.defaultdict(list) for task_entity in task_entities: task_entities_by_parent_id[task_entity["parent_id"]].append( task_entity ) + missing_keys_by_object_name = collections.defaultdict(set) for parent_id, values in interesting_data.items(): - task_entities = task_entities_by_parent_id[parent_id] + entities = task_entities_by_parent_id.get(parent_id) or [] + entities.append(entities_by_id[parent_id]) + for key, value in values.items(): changed_ids = [] - for task_entity in task_entities: - task_id = task_entity["id"] - changed_ids.append(task_id) + for entity in entities: + entity_attrs_mapping = ( + attrs_by_obj_id.get(entity["object_type_id"]) + ) + if not entity_attrs_mapping: + missing_keys_by_object_name[entity.entity_key].add( + key + ) + continue + configuration_id = entity_attrs_mapping.get(key) + if not configuration_id: + missing_keys_by_object_name[entity.entity_key].add( + key + ) + continue + + changed_ids.append(entity["id"]) entity_key = collections.OrderedDict({ - "configuration_id": task_attr_id_by_key[key], - "entity_id": task_id + "configuration_id": configuration_id, + "entity_id": entity["id"] }) if value is None: op = ftrack_api.operation.DeleteEntityOperation( @@ -142,6 +212,19 @@ class PushFrameValuesToTaskEvent(BaseEvent): "Changing of values failed.", exc_info=True ) + if not missing_keys_by_object_name: + return + + msg_items = [] + for object_name, missing_keys in missing_keys_by_object_name.items(): + msg_items.append( + "{}: ({})".format(object_name, self.join_keys(missing_keys)) + ) + + self.log.warning(( + "Missing Custom Attribute configuration" + " per specific object types: {}" + ).format(", ".join(msg_items))) def register(session, plugins_presets): From a628260c82d6c787960df501560fea8999acbc16 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 20 Aug 2020 19:33:42 +0200 Subject: [PATCH 10/13] moved code in better order --- .../events/event_push_frame_values_to_task.py | 144 +++++++++--------- 1 file changed, 72 insertions(+), 72 deletions(-) diff --git a/pype/modules/ftrack/events/event_push_frame_values_to_task.py b/pype/modules/ftrack/events/event_push_frame_values_to_task.py index dd24110c1b..d4056c2ae5 100644 --- a/pype/modules/ftrack/events/event_push_frame_values_to_task.py +++ b/pype/modules/ftrack/events/event_push_frame_values_to_task.py @@ -49,78 +49,6 @@ class PushFrameValuesToTaskEvent(BaseEvent): ) return cls._cached_interest_object_ids - def extract_interesting_data(self, session, event): - # Filter if event contain relevant data - entities_info = event["data"].get("entities") - if not entities_info: - return - - interesting_data = {} - for entity_info in entities_info: - # Care only about tasks - if entity_info.get("entityType") != "task": - continue - - # Care only about changes of status - changes = entity_info.get("changes") or {} - if not changes: - continue - - # Care only about changes if specific keys - entity_changes = {} - for key in self.interest_attributes: - if key in changes: - entity_changes[key] = changes[key]["new"] - - if not entity_changes: - continue - - # Do not care about "Task" entity_type - task_object_id = self.task_object_id(session) - if entity_info.get("objectTypeId") == task_object_id: - continue - - interesting_data[entity_info["entityId"]] = entity_changes - return interesting_data - - def get_entities(self, session, interesting_data): - entities = session.query( - "TypedContext where id in ({})".format( - self.join_keys(interesting_data.keys()) - ) - ).all() - - output = [] - interest_object_ids = self.interest_object_ids(session) - for entity in entities: - if entity["object_type_id"] in interest_object_ids: - output.append(entity) - return output - - def get_task_entities(self, session, interesting_data): - return session.query( - "Task where parent_id in ({})".format( - self.join_keys(interesting_data.keys()) - ) - ).all() - - def attrs_configurations(self, session): - object_ids = list(self.interest_object_ids(session)) - object_ids.append(self.task_object_id(session)) - - attrs = session.query(self.cust_attrs_query.format( - self.join_keys(self.interest_attr_mapping.values()), - self.join_keys(object_ids) - )).all() - - output = {} - for attr in attrs: - obj_id = attr["object_type_id"] - if obj_id not in output: - output[obj_id] = {} - output[obj_id][attr["key"]] = attr["id"] - return output - def launch(self, session, event): interesting_data = self.extract_interesting_data(session, event) if not interesting_data: @@ -226,6 +154,78 @@ class PushFrameValuesToTaskEvent(BaseEvent): " per specific object types: {}" ).format(", ".join(msg_items))) + def extract_interesting_data(self, session, event): + # Filter if event contain relevant data + entities_info = event["data"].get("entities") + if not entities_info: + return + + interesting_data = {} + for entity_info in entities_info: + # Care only about tasks + if entity_info.get("entityType") != "task": + continue + + # Care only about changes of status + changes = entity_info.get("changes") or {} + if not changes: + continue + + # Care only about changes if specific keys + entity_changes = {} + for key in self.interest_attributes: + if key in changes: + entity_changes[key] = changes[key]["new"] + + if not entity_changes: + continue + + # Do not care about "Task" entity_type + task_object_id = self.task_object_id(session) + if entity_info.get("objectTypeId") == task_object_id: + continue + + interesting_data[entity_info["entityId"]] = entity_changes + return interesting_data + + def get_entities(self, session, interesting_data): + entities = session.query( + "TypedContext where id in ({})".format( + self.join_keys(interesting_data.keys()) + ) + ).all() + + output = [] + interest_object_ids = self.interest_object_ids(session) + for entity in entities: + if entity["object_type_id"] in interest_object_ids: + output.append(entity) + return output + + def get_task_entities(self, session, interesting_data): + return session.query( + "Task where parent_id in ({})".format( + self.join_keys(interesting_data.keys()) + ) + ).all() + + def attrs_configurations(self, session): + object_ids = list(self.interest_object_ids(session)) + object_ids.append(self.task_object_id(session)) + + attrs = session.query(self.cust_attrs_query.format( + self.join_keys(self.interest_attr_mapping.values()), + self.join_keys(object_ids) + )).all() + + output = {} + for attr in attrs: + obj_id = attr["object_type_id"] + if obj_id not in output: + output[obj_id] = {} + output[obj_id][attr["key"]] = attr["id"] + return output + def register(session, plugins_presets): PushFrameValuesToTaskEvent(session, plugins_presets).register() From 293ceb8e0bffda6dd4f1975633c915f5457ccb6f Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 20 Aug 2020 19:33:53 +0200 Subject: [PATCH 11/13] fixed few minor bugs --- .../ftrack/events/event_push_frame_values_to_task.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pype/modules/ftrack/events/event_push_frame_values_to_task.py b/pype/modules/ftrack/events/event_push_frame_values_to_task.py index d4056c2ae5..32993ef938 100644 --- a/pype/modules/ftrack/events/event_push_frame_values_to_task.py +++ b/pype/modules/ftrack/events/event_push_frame_values_to_task.py @@ -67,8 +67,6 @@ class PushFrameValuesToTaskEvent(BaseEvent): interesting_data.pop(entity_id) task_entities = self.get_task_entities(session, interesting_data) - if not task_entities: - return attrs_by_obj_id = self.attrs_configurations(session) if not attrs_by_obj_id: @@ -89,21 +87,22 @@ class PushFrameValuesToTaskEvent(BaseEvent): entities = task_entities_by_parent_id.get(parent_id) or [] entities.append(entities_by_id[parent_id]) - for key, value in values.items(): + for hier_key, value in values.items(): changed_ids = [] for entity in entities: + key = self.interest_attr_mapping[hier_key] entity_attrs_mapping = ( attrs_by_obj_id.get(entity["object_type_id"]) ) if not entity_attrs_mapping: - missing_keys_by_object_name[entity.entity_key].add( + missing_keys_by_object_name[entity.entity_type].add( key ) continue configuration_id = entity_attrs_mapping.get(key) if not configuration_id: - missing_keys_by_object_name[entity.entity_key].add( + missing_keys_by_object_name[entity.entity_type].add( key ) continue From 009a02f104019c64e94a7cacc4b7c8748c56d018 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 20 Aug 2020 19:39:49 +0200 Subject: [PATCH 12/13] removed unnecessary logs --- .../ftrack/events/action_push_frame_values_to_task.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pype/modules/ftrack/events/action_push_frame_values_to_task.py b/pype/modules/ftrack/events/action_push_frame_values_to_task.py index d88f4a1016..dec34a58cb 100644 --- a/pype/modules/ftrack/events/action_push_frame_values_to_task.py +++ b/pype/modules/ftrack/events/action_push_frame_values_to_task.py @@ -326,13 +326,7 @@ class PushFrameValuesToTaskAction(BaseAction): missing_keys = set() total_parents = len(hier_values_by_entity_id) - idx = 0 for parent_id, values in hier_values_by_entity_id.items(): - idx += 1 - self.log.info(( - "[{}/{}] {} Processing values to task. Values: {}" - ).format(idx, total_parents, parent_id, values)) - task_entities = task_entities_by_parent_id[parent_id] for hier_key, value in values.items(): key = self.custom_attribute_mapping[hier_key] From 576beb744687294bd12f48ce64d7a9ac8d1361bf Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 20 Aug 2020 20:34:50 +0200 Subject: [PATCH 13/13] removed unused variable --- pype/modules/ftrack/events/action_push_frame_values_to_task.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pype/modules/ftrack/events/action_push_frame_values_to_task.py b/pype/modules/ftrack/events/action_push_frame_values_to_task.py index dec34a58cb..a55c1e46a6 100644 --- a/pype/modules/ftrack/events/action_push_frame_values_to_task.py +++ b/pype/modules/ftrack/events/action_push_frame_values_to_task.py @@ -324,8 +324,6 @@ class PushFrameValuesToTaskAction(BaseAction): task_attr_id_by_keys ): missing_keys = set() - - total_parents = len(hier_values_by_entity_id) for parent_id, values in hier_values_by_entity_id.items(): task_entities = task_entities_by_parent_id[parent_id] for hier_key, value in values.items():