diff --git a/pype/hosts/maya/plugins/publish/extract_playblast.py b/pype/hosts/maya/plugins/publish/extract_playblast.py index 99411e7f53..8402e41285 100644 --- a/pype/hosts/maya/plugins/publish/extract_playblast.py +++ b/pype/hosts/maya/plugins/publish/extract_playblast.py @@ -118,7 +118,8 @@ class ExtractPlayblast(pype.api.Extractor): tags.append("delete") # Add camera node name to representation data - camera_node_name = pm.ls(camera)[0].getTransform().getName() + camera_node_name = pm.ls(camera)[0].getTransform().name() + representation = { 'name': 'png', diff --git a/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py b/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py index 7c9c4d196f..be3a15b049 100644 --- a/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py +++ b/pype/modules/ftrack/event_handlers_server/event_sync_to_avalon.py @@ -1,6 +1,7 @@ import os import collections import copy +import json import queue import time import datetime @@ -10,6 +11,7 @@ import traceback from bson.objectid import ObjectId from pymongo import UpdateOne +import arrow import ftrack_api from avalon import schema @@ -31,6 +33,15 @@ class SyncToAvalonEvent(BaseEvent): ignore_ent_types = ["Milestone"] ignore_keys = ["statusid", "thumbid"] + cust_attr_query_keys = [ + "id", + "key", + "entity_type", + "object_type_id", + "is_hierarchical", + "config", + "default" + ] project_query = ( "select full_name, name, custom_attributes" ", project_schema._task_type_schema.types.name" @@ -115,10 +126,22 @@ class SyncToAvalonEvent(BaseEvent): def avalon_cust_attrs(self): if self._avalon_cust_attrs is None: self._avalon_cust_attrs = avalon_sync.get_pype_attr( - self.process_session + self.process_session, query_keys=self.cust_attr_query_keys ) return self._avalon_cust_attrs + @property + def cust_attr_types_by_id(self): + if self._cust_attr_types_by_id is None: + cust_attr_types = self.process_session.query( + "select id, name from CustomAttributeType" + ).all() + self._cust_attr_types_by_id = { + cust_attr_type["id"]: cust_attr_type + for cust_attr_type in cust_attr_types + } + return self._cust_attr_types_by_id + @property def avalon_entities(self): if self._avalon_ents is None: @@ -227,15 +250,6 @@ class SyncToAvalonEvent(BaseEvent): return self._changeability_by_mongo_id - @property - def avalon_custom_attributes(self): - """Return info about changeability of entity and it's parents.""" - if self._avalon_custom_attributes is None: - self._avalon_custom_attributes = avalon_sync.get_pype_attr( - self.process_session - ) - return self._avalon_custom_attributes - def remove_cached_by_key(self, key, values): if self._avalon_ents is None: return @@ -380,6 +394,7 @@ class SyncToAvalonEvent(BaseEvent): self._cur_project = None self._avalon_cust_attrs = None + self._cust_attr_types_by_id = None self._avalon_ents = None self._avalon_ents_by_id = None @@ -391,7 +406,6 @@ class SyncToAvalonEvent(BaseEvent): self._avalon_archived_by_id = None self._avalon_archived_by_name = None - self._avalon_custom_attributes = None self._ent_types_by_name = None self.ftrack_ents_by_id = {} @@ -1234,54 +1248,28 @@ class SyncToAvalonEvent(BaseEvent): return final_entity - def get_cust_attr_values(self, entity, keys=None): + def get_cust_attr_values(self, entity): output = {} - custom_attrs, hier_attrs = self.avalon_custom_attributes - not_processed_keys = True - if keys: - not_processed_keys = [k for k in keys] + custom_attrs, hier_attrs = self.avalon_cust_attrs + # Notmal custom attributes - processed_keys = [] for attr in custom_attrs: - if not not_processed_keys: - break key = attr["key"] - if key in processed_keys: - continue + if key in entity["custom_attributes"]: + output[key] = entity["custom_attributes"][key] - if key not in entity["custom_attributes"]: - continue - - if keys: - if key not in keys: - continue - else: - not_processed_keys.remove(key) - - output[key] = entity["custom_attributes"][key] - processed_keys.append(key) - - if not not_processed_keys: - return output - - # Hierarchical cust attrs - hier_keys = [] - defaults = {} - for attr in hier_attrs: - key = attr["key"] - if keys and key not in keys: - continue - hier_keys.append(key) - defaults[key] = attr["default"] - - hier_values = avalon_sync.get_hierarchical_attributes( - self.process_session, entity, hier_keys, defaults + hier_values = avalon_sync.get_hierarchical_attributes_values( + self.process_session, + entity, + hier_attrs, + self.cust_attr_types_by_id ) for key, val in hier_values.items(): - if key == CUST_ATTR_ID_KEY: - continue output[key] = val + # Make sure mongo id is not set + output.pop(CUST_ATTR_ID_KEY, None) + return output def process_renamed(self): @@ -1548,10 +1536,9 @@ class SyncToAvalonEvent(BaseEvent): ).format(entity_type, ent_info["entityType"])) continue - _entity_key = collections.OrderedDict({ - "configuration_id": mongo_id_configuration_id, - "entity_id": ftrack_id - }) + _entity_key = collections.OrderedDict() + _entity_key["configuration_id"] = mongo_id_configuration_id + _entity_key["entity_id"] = ftrack_id self.process_session.recorded_operations.push( ftrack_api.operation.UpdateEntityOperation( @@ -1790,6 +1777,10 @@ class SyncToAvalonEvent(BaseEvent): return cust_attrs, hier_attrs = self.avalon_cust_attrs + hier_attrs_by_key = { + attr["key"]: attr + for attr in hier_attrs + } cust_attrs_by_obj_id = collections.defaultdict(dict) for cust_attr in cust_attrs: key = cust_attr["key"] @@ -1805,8 +1796,6 @@ class SyncToAvalonEvent(BaseEvent): obj_id = cust_attr["object_type_id"] cust_attrs_by_obj_id[obj_id][key] = cust_attr - hier_attrs_keys = [attr["key"] for attr in hier_attrs] - for ftrack_id, ent_info in ent_infos.items(): mongo_id = ftrack_mongo_mapping[ftrack_id] entType = ent_info["entityType"] @@ -1819,44 +1808,79 @@ class SyncToAvalonEvent(BaseEvent): # Ftrack's entity_type does not have defined custom attributes if ent_cust_attrs is None: - ent_cust_attrs = [] + ent_cust_attrs = {} for key, values in ent_info["changes"].items(): - if key in hier_attrs_keys: + if key in hier_attrs_by_key: self.hier_cust_attrs_changes[key].append(ftrack_id) continue if key not in ent_cust_attrs: continue + value = values["new"] + new_value = self.convert_value_by_cust_attr_conf( + value, ent_cust_attrs[key] + ) + + if entType == "show" and key == "applications": + # Store apps to project't config + proj_apps, warnings = ( + avalon_sync.get_project_apps(new_value) + ) + if "config" not in self.updates[mongo_id]: + self.updates[mongo_id]["config"] = {} + self.updates[mongo_id]["config"]["apps"] = proj_apps + + for msg, items in warnings.items(): + if not msg or not items: + continue + self.report_items["warning"][msg] = items + continue + if "data" not in self.updates[mongo_id]: self.updates[mongo_id]["data"] = {} - value = values["new"] - self.updates[mongo_id]["data"][key] = value + self.updates[mongo_id]["data"][key] = new_value self.log.debug( "Setting data value of \"{}\" to \"{}\" <{}>".format( - key, value, ent_path + key, new_value, ent_path ) ) - if entType != "show" or key != "applications": - continue + def convert_value_by_cust_attr_conf(self, value, cust_attr_conf): + type_id = cust_attr_conf["type_id"] + cust_attr_type_name = self.cust_attr_types_by_id[type_id]["name"] + ignored = ( + "expression", "notificationtype", "dynamic enumerator" + ) + if cust_attr_type_name in ignored: + return None - # Store apps to project't config - apps_str = ent_info["changes"]["applications"]["new"] - cust_attr_apps = [app for app in apps_str.split(", ") if app] + if cust_attr_type_name == "text": + return value - proj_apps, warnings = ( - avalon_sync.get_project_apps(cust_attr_apps) - ) - if "config" not in self.updates[mongo_id]: - self.updates[mongo_id]["config"] = {} - self.updates[mongo_id]["config"]["apps"] = proj_apps + if cust_attr_type_name == "boolean": + if value == "1": + return True + if value == "0": + return False + return bool(value) - for msg, items in warnings.items(): - if not msg or not items: - continue - self.report_items["warning"][msg] = items + if cust_attr_type_name == "date": + return arrow.get(value) + + cust_attr_config = json.loads(cust_attr_conf["config"]) + + if cust_attr_type_name == "number": + if cust_attr_config["isdecimal"]: + return float(value) + return int(value) + + if cust_attr_type_name == "enumerator": + if not cust_attr_config["multiSelect"]: + return value + return value.split(", ") + return value def process_hier_cleanup(self): if ( @@ -1968,7 +1992,7 @@ class SyncToAvalonEvent(BaseEvent): self.update_entities() return - cust_attrs, hier_attrs = self.avalon_cust_attrs + _, hier_attrs = self.avalon_cust_attrs # Hierarchical custom attributes preparation *** hier_attr_key_by_id = { @@ -2087,21 +2111,18 @@ class SyncToAvalonEvent(BaseEvent): parent_queue.put(parent_ent) # Prepare values to query - entity_ids_joined = ", ".join([ - "\"{}\"".format(id) for id in cust_attrs_ftrack_ids - ]) configuration_ids = set() for key in hier_cust_attrs_keys: configuration_ids.add(hier_attr_id_by_key[key]) - attributes_joined = ", ".join([ - "\"{}\"".format(conf_id) for conf_id in configuration_ids - ]) + entity_ids_joined = self.join_query_keys(cust_attrs_ftrack_ids) + attributes_joined = self.join_query_keys(configuration_ids) queries = [{ "action": "query", "expression": ( - "select value, entity_id from CustomAttributeValue " + "select value, entity_id, configuration_id" + " from CustomAttributeValue " "where entity_id in ({}) and configuration_id in ({})" ).format(entity_ids_joined, attributes_joined) }] @@ -2113,22 +2134,40 @@ class SyncToAvalonEvent(BaseEvent): ftrack_project_id = self.cur_project["id"] + attr_types_by_id = self.cust_attr_types_by_id + convert_types_by_id = {} for attr in hier_attrs: key = attr["key"] if key not in hier_cust_attrs_keys: continue + + type_id = attr["type_id"] + attr_id = attr["id"] + cust_attr_type_name = attr_types_by_id[type_id]["name"] + convert_type = avalon_sync.get_python_type_for_custom_attribute( + attr, cust_attr_type_name + ) + + convert_types_by_id[attr_id] = convert_type entities_dict[ftrack_project_id]["hier_attrs"][key] = ( attr["default"] ) # PREPARE DATA BEFORE THIS avalon_hier = [] - for value in values["data"]: - if value["value"] is None: + for item in values["data"]: + value = item["value"] + if value is None: continue - entity_id = value["entity_id"] - key = hier_attr_key_by_id[value["configuration_id"]] - entities_dict[entity_id]["hier_attrs"][key] = value["value"] + entity_id = item["entity_id"] + configuration_id = item["configuration_id"] + + convert_type = convert_types_by_id[configuration_id] + key = hier_attr_key_by_id[configuration_id] + + if convert_type: + value = convert_type(value) + entities_dict[entity_id]["hier_attrs"][key] = value # Get dictionary with not None hierarchical values to pull to childs project_values = {} diff --git a/pype/modules/ftrack/lib/avalon_sync.py b/pype/modules/ftrack/lib/avalon_sync.py index d639e814a5..a6151bfba9 100644 --- a/pype/modules/ftrack/lib/avalon_sync.py +++ b/pype/modules/ftrack/lib/avalon_sync.py @@ -83,15 +83,27 @@ def check_regex(name, entity_type, in_schema=None, schema_patterns=None): return False -def get_pype_attr(session, split_hierarchical=True): +def join_query_keys(keys): + return ",".join(["\"{}\"".format(key) for key in keys]) + + +def get_pype_attr(session, split_hierarchical=True, query_keys=None): custom_attributes = [] hier_custom_attributes = [] + if not query_keys: + query_keys = [ + "id", + "entity_type", + "object_type_id", + "is_hierarchical", + "default" + ] # TODO remove deprecated "avalon" group from query cust_attrs_query = ( - "select id, entity_type, object_type_id, is_hierarchical, default" + "select {}" " from CustomAttributeConfiguration" - " where group.name in (\"avalon\", \"pype\")" - ) + " where group.name in (\"avalon\", \"{}\")" + ).format(", ".join(query_keys), CUST_ATTR_GROUP) all_avalon_attr = session.query(cust_attrs_query).all() for cust_attr in all_avalon_attr: if split_hierarchical and cust_attr["is_hierarchical"]: @@ -107,6 +119,39 @@ def get_pype_attr(session, split_hierarchical=True): return custom_attributes +def get_python_type_for_custom_attribute(cust_attr, cust_attr_type_name=None): + """Python type that should value of custom attribute have. + + This function is mainly for number type which is always float from ftrack. + + Returns: + type: Python type which call be called on object to convert the object + to the type or None if can't figure out. + """ + if cust_attr_type_name is None: + cust_attr_type_name = cust_attr["type"]["name"] + + if cust_attr_type_name == "text": + return str + + if cust_attr_type_name == "boolean": + return bool + + if cust_attr_type_name in ("number", "enumerator"): + cust_attr_config = json.loads(cust_attr["config"]) + if cust_attr_type_name == "number": + if cust_attr_config["isdecimal"]: + return float + return int + + if cust_attr_type_name == "enumerator": + if cust_attr_config["multiSelect"]: + return list + return str + # "date", "expression", "notificationtype", "dynamic enumerator" + return None + + def from_dict_to_set(data, is_project): """ Converts 'data' into $set part of MongoDB update command. @@ -191,93 +236,97 @@ def get_project_apps(in_app_list): apps = [] warnings = collections.defaultdict(list) + if not in_app_list: + return apps, warnings + missing_app_msg = "Missing definition of application" application_manager = ApplicationManager() for app_name in in_app_list: - app = application_manager.applications.get(app_name) - if app: - apps.append({ - "name": app_name, - "label": app.full_label - }) + if application_manager.applications.get(app_name): + apps.append({"name": app_name}) else: warnings[missing_app_msg].append(app_name) return apps, warnings -def get_hierarchical_attributes(session, entity, attr_names, attr_defaults={}): - entity_ids = [] - if entity.entity_type.lower() == "project": - entity_ids.append(entity["id"]) - else: - typed_context = session.query(( - "select ancestors.id, project from TypedContext where id is \"{}\"" - ).format(entity["id"])).one() - entity_ids.append(typed_context["id"]) - entity_ids.extend( - [ent["id"] for ent in reversed(typed_context["ancestors"])] +def get_hierarchical_attributes_values( + session, entity, hier_attrs, cust_attr_types=None +): + if not cust_attr_types: + cust_attr_types = session.query( + "select id, name from CustomAttributeType" + ).all() + + cust_attr_name_by_id = { + cust_attr_type["id"]: cust_attr_type["name"] + for cust_attr_type in cust_attr_types + } + # Hierarchical cust attrs + attr_key_by_id = {} + convert_types_by_attr_id = {} + defaults = {} + for attr in hier_attrs: + attr_id = attr["id"] + key = attr["key"] + type_id = attr["type_id"] + + attr_key_by_id[attr_id] = key + defaults[key] = attr["default"] + + cust_attr_type_name = cust_attr_name_by_id[type_id] + convert_type = get_python_type_for_custom_attribute( + attr, cust_attr_type_name ) - entity_ids.append(typed_context["project"]["id"]) + convert_types_by_attr_id[attr_id] = convert_type - missing_defaults = [] - for attr_name in attr_names: - if attr_name not in attr_defaults: - missing_defaults.append(attr_name) + entity_ids = [item["id"] for item in entity["link"]] + + join_ent_ids = join_query_keys(entity_ids) + join_attribute_ids = join_query_keys(attr_key_by_id.keys()) - join_ent_ids = ", ".join( - ["\"{}\"".format(entity_id) for entity_id in entity_ids] - ) - join_attribute_names = ", ".join( - ["\"{}\"".format(key) for key in attr_names] - ) queries = [] queries.append({ "action": "query", "expression": ( - "select value, entity_id from CustomAttributeValue " - "where entity_id in ({}) and configuration.key in ({})" - ).format(join_ent_ids, join_attribute_names) + "select value, configuration_id, entity_id" + " from CustomAttributeValue" + " where entity_id in ({}) and configuration_id in ({})" + ).format(join_ent_ids, join_attribute_ids) }) - if not missing_defaults: - if hasattr(session, "call"): - [values] = session.call(queries) - else: - [values] = session._call(queries) + if hasattr(session, "call"): + [values] = session.call(queries) else: - join_missing_names = ", ".join( - ["\"{}\"".format(key) for key in missing_defaults] - ) - queries.append({ - "action": "query", - "expression": ( - "select default from CustomAttributeConfiguration " - "where key in ({})" - ).format(join_missing_names) - }) - - [values, default_values] = session.call(queries) - for default_value in default_values: - key = default_value["data"][0]["key"] - attr_defaults[key] = default_value["data"][0]["default"] + [values] = session._call(queries) hier_values = {} - for key, val in attr_defaults.items(): + for key, val in defaults.items(): hier_values[key] = val if not values["data"]: return hier_values - _hier_values = collections.defaultdict(list) - for value in values["data"]: - key = value["configuration"]["key"] - _hier_values[key].append(value) + values_by_entity_id = collections.defaultdict(dict) + for item in values["data"]: + value = item["value"] + if value is None: + continue - for key, values in _hier_values.items(): - value = sorted( - values, key=lambda value: entity_ids.index(value["entity_id"]) - )[0] - hier_values[key] = value["value"] + attr_id = item["configuration_id"] + + convert_type = convert_types_by_attr_id[attr_id] + if convert_type: + value = convert_type(value) + + key = attr_key_by_id[attr_id] + entity_id = item["entity_id"] + values_by_entity_id[entity_id][key] = value + + for entity_id in entity_ids: + for key in attr_key_by_id.values(): + value = values_by_entity_id[entity_id].get(key) + if value is not None: + hier_values[key] = value return hier_values @@ -285,6 +334,16 @@ def get_hierarchical_attributes(session, entity, attr_names, attr_defaults={}): class SyncEntitiesFactory: dbcon = AvalonMongoDB() + cust_attr_query_keys = [ + "id", + "key", + "entity_type", + "object_type_id", + "is_hierarchical", + "config", + "default" + ] + project_query = ( "select full_name, name, custom_attributes" ", project_schema._task_type_schema.types.name" @@ -830,11 +889,21 @@ class SyncEntitiesFactory: def set_cutom_attributes(self): self.log.debug("* Preparing custom attributes") # Get custom attributes and values - custom_attrs, hier_attrs = get_pype_attr(self.session) + custom_attrs, hier_attrs = get_pype_attr( + self.session, query_keys=self.cust_attr_query_keys + ) ent_types = self.session.query("select id, name from ObjectType").all() ent_types_by_name = { ent_type["name"]: ent_type["id"] for ent_type in ent_types } + # Custom attribute types + cust_attr_types = self.session.query( + "select id, name from CustomAttributeType" + ).all() + cust_attr_type_name_by_id = { + cust_attr_type["id"]: cust_attr_type["name"] + for cust_attr_type in cust_attr_types + } # store default values per entity type attrs_per_entity_type = collections.defaultdict(dict) @@ -844,9 +913,20 @@ class SyncEntitiesFactory: avalon_attrs_ca_id = collections.defaultdict(dict) attribute_key_by_id = {} + convert_types_by_attr_id = {} for cust_attr in custom_attrs: key = cust_attr["key"] - attribute_key_by_id[cust_attr["id"]] = key + attr_id = cust_attr["id"] + type_id = cust_attr["type_id"] + + attribute_key_by_id[attr_id] = key + cust_attr_type_name = cust_attr_type_name_by_id[type_id] + + convert_type = get_python_type_for_custom_attribute( + cust_attr, cust_attr_type_name + ) + convert_types_by_attr_id[attr_id] = convert_type + ca_ent_type = cust_attr["entity_type"] if key.startswith("avalon_"): if ca_ent_type == "show": @@ -924,8 +1004,9 @@ class SyncEntitiesFactory: ]) cust_attr_query = ( - "select value, entity_id from ContextCustomAttributeValue " - "where entity_id in ({}) and configuration_id in ({})" + "select value, configuration_id, entity_id" + " from ContextCustomAttributeValue" + " where entity_id in ({}) and configuration_id in ({})" ) call_expr = [{ "action": "query", @@ -940,24 +1021,44 @@ class SyncEntitiesFactory: for item in values["data"]: entity_id = item["entity_id"] - key = attribute_key_by_id[item["configuration_id"]] + attr_id = item["configuration_id"] + key = attribute_key_by_id[attr_id] store_key = "custom_attributes" if key.startswith("avalon_"): store_key = "avalon_attrs" - self.entities_dict[entity_id][store_key][key] = item["value"] + + convert_type = convert_types_by_attr_id[attr_id] + value = item["value"] + if convert_type: + value = convert_type(value) + self.entities_dict[entity_id][store_key][key] = value # process hierarchical attributes - self.set_hierarchical_attribute(hier_attrs, sync_ids) + self.set_hierarchical_attribute( + hier_attrs, sync_ids, cust_attr_type_name_by_id + ) - def set_hierarchical_attribute(self, hier_attrs, sync_ids): + def set_hierarchical_attribute( + self, hier_attrs, sync_ids, cust_attr_type_name_by_id + ): # collect all hierarchical attribute keys # and prepare default values to project attributes_by_key = {} attribute_key_by_id = {} + convert_types_by_attr_id = {} for attr in hier_attrs: key = attr["key"] - attribute_key_by_id[attr["id"]] = key + attr_id = attr["id"] + type_id = attr["type_id"] + attribute_key_by_id[attr_id] = key attributes_by_key[key] = attr + + cust_attr_type_name = cust_attr_type_name_by_id[type_id] + convert_type = get_python_type_for_custom_attribute( + attr, cust_attr_type_name + ) + convert_types_by_attr_id[attr_id] = convert_type + self.hier_cust_attr_ids_by_key[key] = attr["id"] store_key = "hier_attrs" @@ -992,7 +1093,7 @@ class SyncEntitiesFactory: else: prepare_dict[key] = None - for id, entity_dict in self.entities_dict.items(): + for entity_dict in self.entities_dict.values(): # Skip project because has stored defaults at the moment if entity_dict["entity_type"] == "project": continue @@ -1011,8 +1112,9 @@ class SyncEntitiesFactory: call_expr = [{ "action": "query", "expression": ( - "select value, entity_id from ContextCustomAttributeValue " - "where entity_id in ({}) and configuration_id in ({})" + "select value, entity_id, configuration_id" + " from ContextCustomAttributeValue" + " where entity_id in ({}) and configuration_id in ({})" ).format(entity_ids_joined, attributes_joined) }] if hasattr(self.session, "call"): @@ -1030,8 +1132,14 @@ class SyncEntitiesFactory: or (isinstance(value, (tuple, list)) and not value) ): continue + + attr_id = item["configuration_id"] + convert_type = convert_types_by_attr_id[attr_id] + if convert_type: + value = convert_type(value) + entity_id = item["entity_id"] - key = attribute_key_by_id[item["configuration_id"]] + key = attribute_key_by_id[attr_id] if key.startswith("avalon_"): store_key = "avalon_attrs" avalon_hier.append(key) @@ -1141,7 +1249,7 @@ class SyncEntitiesFactory: proj_schema = entity["project_schema"] task_types = proj_schema["_task_type_schema"]["types"] proj_apps, warnings = get_project_apps( - (data.get("applications") or []) + data.pop("applications", []) ) for msg, items in warnings.items(): if not msg or not items: @@ -1428,8 +1536,13 @@ class SyncEntitiesFactory: old_parent_name = self.entities_dict[ self.ft_project_id]["name"] else: - old_parent_name = self.avalon_ents_by_id[ - ftrack_parent_mongo_id]["name"] + old_parent_name = "N/A" + if ftrack_parent_mongo_id in self.avalon_ents_by_id: + old_parent_name = ( + self.avalon_ents_by_id + [ftrack_parent_mongo_id] + ["name"] + ) self.updates[avalon_id]["data"] = { "visualParent": new_parent_id @@ -2139,11 +2252,22 @@ class SyncEntitiesFactory: final_doc_data = self.entities_dict[self.ft_project_id]["final_entity"] final_doc_tasks = final_doc_data["config"].pop("tasks") current_doc_tasks = self.avalon_project.get("config", {}).get("tasks") - # Update project's tasks if tasks are empty or are not same - if not final_doc_tasks: + # Update project's task types + if not current_doc_tasks: update_tasks = True else: - update_tasks = final_doc_tasks != current_doc_tasks + # Check if task types are same + update_tasks = False + for task_type in final_doc_tasks: + if task_type not in current_doc_tasks: + update_tasks = True + break + + # Update new task types + # - but keep data about existing types and only add new one + if update_tasks: + for task_type, type_data in current_doc_tasks.items(): + final_doc_tasks[task_type] = type_data changes = self.compare_dict(final_doc_data, self.avalon_project) @@ -2372,7 +2496,7 @@ class SyncEntitiesFactory: if new_entity_id not in p_chilren: self.entities_dict[parent_id]["children"].append(new_entity_id) - cust_attr, hier_attrs = get_pype_attr(self.session) + cust_attr, _ = get_pype_attr(self.session) for _attr in cust_attr: key = _attr["key"] if key not in av_entity["data"]: diff --git a/pype/modules/idle_manager/__init__.py b/pype/modules/idle_manager/__init__.py index 4bc33c87c1..651f360c50 100644 --- a/pype/modules/idle_manager/__init__.py +++ b/pype/modules/idle_manager/__init__.py @@ -1,4 +1,4 @@ -from .idle_manager import ( +from .idle_module import ( IdleManager, IIdleManager ) diff --git a/pype/modules/idle_manager/idle_module.py b/pype/modules/idle_manager/idle_module.py new file mode 100644 index 0000000000..979e1b92ea --- /dev/null +++ b/pype/modules/idle_manager/idle_module.py @@ -0,0 +1,97 @@ +import collections +from abc import ABCMeta, abstractmethod + +import six + +from pype.modules import PypeModule, ITrayService + + +@six.add_metaclass(ABCMeta) +class IIdleManager: + """Other modules interface to return callbacks by idle time in seconds. + + Expected output is dictionary with seconds as keys and callback/s + as value, value may be callback of list of callbacks. + EXAMPLE: + ``` + { + 60: self.on_minute_idle + } + ``` + """ + idle_manager = None + + @abstractmethod + def callbacks_by_idle_time(self): + pass + + @property + def idle_time(self): + if self.idle_manager: + return self.idle_manager.idle_time + + +class IdleManager(PypeModule, ITrayService): + """ Measure user's idle time in seconds. + Idle time resets on keyboard/mouse input. + Is able to emit signals at specific time idle. + """ + label = "Idle Service" + name = "idle_manager" + + def initialize(self, module_settings): + idle_man_settings = module_settings[self.name] + self.enabled = idle_man_settings["enabled"] + + self.time_callbacks = collections.defaultdict(list) + self.idle_thread = None + + def tray_init(self): + return + + def tray_start(self): + self.start_thread() + + def tray_exit(self): + self.stop_thread() + try: + self.time_callbacks = {} + except Exception: + pass + + def connect_with_modules(self, enabled_modules): + for module in enabled_modules: + if not isinstance(module, IIdleManager): + continue + + module.idle_manager = self + callbacks_items = module.callbacks_by_idle_time() or {} + for emit_time, callbacks in callbacks_items.items(): + if not isinstance(callbacks, (tuple, list, set)): + callbacks = [callbacks] + self.time_callbacks[emit_time].extend(callbacks) + + @property + def idle_time(self): + if self.idle_thread and self.idle_thread.is_running: + return self.idle_thread.idle_time + + def _create_thread(self): + from .idle_threads import IdleManagerThread + + return IdleManagerThread(self) + + def start_thread(self): + if self.idle_thread: + self.idle_thread.stop() + self.idle_thread.join() + self.idle_thread = self._create_thread() + self.idle_thread.start() + + def stop_thread(self): + if self.idle_thread: + self.idle_thread.stop() + self.idle_thread.join() + + def on_thread_stop(self): + self.set_service_failed_icon() diff --git a/pype/modules/idle_manager/idle_manager.py b/pype/modules/idle_manager/idle_threads.py similarity index 50% rename from pype/modules/idle_manager/idle_manager.py rename to pype/modules/idle_manager/idle_threads.py index fa6d70d229..7cedf986e6 100644 --- a/pype/modules/idle_manager/idle_manager.py +++ b/pype/modules/idle_manager/idle_threads.py @@ -1,105 +1,38 @@ import time -import collections import threading -from abc import ABCMeta, abstractmethod -import six from pynput import mouse, keyboard from pype.lib import PypeLogger -from pype.modules import PypeModule, ITrayService -@six.add_metaclass(ABCMeta) -class IIdleManager: - """Other modules interface to return callbacks by idle time in seconds. +class MouseThread(mouse.Listener): + """Listens user's mouse movement.""" - Expected output is dictionary with seconds as keys and callback/s - as value, value may be callback of list of callbacks. - EXAMPLE: - ``` - { - 60: self.on_minute_idle - } - ``` - """ - idle_manager = None + def __init__(self, callback): + super(MouseThread, self).__init__(on_move=self.on_move) + self.callback = callback - @abstractmethod - def callbacks_by_idle_time(self): - pass - - @property - def idle_time(self): - if self.idle_manager: - return self.idle_manager.idle_time + def on_move(self, posx, posy): + self.callback() -class IdleManager(PypeModule, ITrayService): - """ Measure user's idle time in seconds. - Idle time resets on keyboard/mouse input. - Is able to emit signals at specific time idle. - """ - label = "Idle Service" - name = "idle_manager" +class KeyboardThread(keyboard.Listener): + """Listens user's keyboard input.""" - def initialize(self, module_settings): - idle_man_settings = module_settings[self.name] - self.enabled = idle_man_settings["enabled"] + def __init__(self, callback): + super(KeyboardThread, self).__init__(on_press=self.on_press) - self.time_callbacks = collections.defaultdict(list) - self.idle_thread = None + self.callback = callback - def tray_init(self): - return - - def tray_start(self): - self.start_thread() - - def tray_exit(self): - self.stop_thread() - try: - self.time_callbacks = {} - except Exception: - pass - - def connect_with_modules(self, enabled_modules): - for module in enabled_modules: - if not isinstance(module, IIdleManager): - continue - - module.idle_manager = self - callbacks_items = module.callbacks_by_idle_time() or {} - for emit_time, callbacks in callbacks_items.items(): - if not isinstance(callbacks, (tuple, list, set)): - callbacks = [callbacks] - self.time_callbacks[emit_time].extend(callbacks) - - @property - def idle_time(self): - if self.idle_thread and self.idle_thread.is_running: - return self.idle_thread.idle_time - - def start_thread(self): - if self.idle_thread: - self.idle_thread.stop() - self.idle_thread.join() - self.idle_thread = IdleManagerThread(self) - self.idle_thread.start() - - def stop_thread(self): - if self.idle_thread: - self.idle_thread.stop() - self.idle_thread.join() - - def on_thread_stop(self): - self.set_service_failed_icon() + def on_press(self, key): + self.callback() class IdleManagerThread(threading.Thread): def __init__(self, module, *args, **kwargs): super(IdleManagerThread, self).__init__(*args, **kwargs) - self.log = PypeLogger().get_logger(self.__class__.__name__) + self.log = PypeLogger.get_logger(self.__class__.__name__) self.module = module self.threads = [] self.is_running = False @@ -124,8 +57,8 @@ class IdleManagerThread(threading.Thread): self.log.info("IdleManagerThread has started") self.is_running = True thread_mouse = MouseThread(self.reset_time) - thread_mouse.start() thread_keyboard = KeyboardThread(self.reset_time) + thread_mouse.start() thread_keyboard.start() try: while self.is_running: @@ -162,26 +95,3 @@ class IdleManagerThread(threading.Thread): pass self.on_stop() - - -class MouseThread(mouse.Listener): - """Listens user's mouse movement.""" - - def __init__(self, callback): - super(MouseThread, self).__init__(on_move=self.on_move) - self.callback = callback - - def on_move(self, posx, posy): - self.callback() - - -class KeyboardThread(keyboard.Listener): - """Listens user's keyboard input.""" - - def __init__(self, callback): - super(KeyboardThread, self).__init__(on_press=self.on_press) - - self.callback = callback - - def on_press(self, key): - self.callback() diff --git a/pype/modules/timers_manager/timers_manager.py b/pype/modules/timers_manager/timers_manager.py index 68890640b3..b83f51f0ba 100644 --- a/pype/modules/timers_manager/timers_manager.py +++ b/pype/modules/timers_manager/timers_manager.py @@ -1,4 +1,5 @@ import os +import collections from abc import ABCMeta, abstractmethod import six from .. import PypeModule, ITrayService, IIdleManager, IWebServerRoutes @@ -159,26 +160,25 @@ class TimersManager(PypeModule, ITrayService, IIdleManager, IWebServerRoutes): def callbacks_by_idle_time(self): """Implementation of IIdleManager interface.""" # Time when message is shown - callbacks = { - self.time_show_message: lambda: self.time_callback(0) - } + callbacks = collections.defaultdict(list) + callbacks[self.time_show_message].append(lambda: self.time_callback(0)) # Times when idle is between show widget and stop timers show_to_stop_range = range( self.time_show_message - 1, self.time_stop_timer ) for num in show_to_stop_range: - callbacks[num] = lambda: self.time_callback(1) + callbacks[num].append(lambda: self.time_callback(1)) # Times when widget is already shown and user restart idle shown_and_moved_range = range( self.time_stop_timer - self.time_show_message ) for num in shown_and_moved_range: - callbacks[num] = lambda: self.time_callback(1) + callbacks[num].append(lambda: self.time_callback(1)) # Time when timers are stopped - callbacks[self.time_stop_timer] = lambda: self.time_callback(2) + callbacks[self.time_stop_timer].append(lambda: self.time_callback(2)) return callbacks diff --git a/pype/modules/timers_manager/widget_user_idle.py b/pype/modules/timers_manager/widget_user_idle.py index 5e47cdaddf..cbdb7fd30a 100644 --- a/pype/modules/timers_manager/widget_user_idle.py +++ b/pype/modules/timers_manager/widget_user_idle.py @@ -163,8 +163,9 @@ class SignalHandler(QtCore.QObject): signal_change_label = QtCore.Signal() signal_stop_timers = QtCore.Signal() - def __init__(self, cls): - super().__init__() - self.signal_show_message.connect(cls.show_message) - self.signal_change_label.connect(cls.change_label) - self.signal_stop_timers.connect(cls.stop_timers) + def __init__(self, module): + super(SignalHandler, self).__init__() + self.module = module + self.signal_show_message.connect(module.show_message) + self.signal_change_label.connect(module.change_label) + self.signal_stop_timers.connect(module.stop_timers) diff --git a/pype/settings/entities/dict_mutable_keys_entity.py b/pype/settings/entities/dict_mutable_keys_entity.py index f930d3738b..c314242bf9 100644 --- a/pype/settings/entities/dict_mutable_keys_entity.py +++ b/pype/settings/entities/dict_mutable_keys_entity.py @@ -7,7 +7,8 @@ from .lib import ( from . import EndpointEntity from .exceptions import ( DefaultsNotDefined, - StudioDefaultsNotDefined + StudioDefaultsNotDefined, + RequiredKeyModified ) from pype.settings.constants import ( METADATA_KEYS, @@ -51,6 +52,8 @@ class DictMutableKeysEntity(EndpointEntity): return key in self.children_by_key def pop(self, key, *args, **kwargs): + if key in self.required_keys: + raise RequiredKeyModified(self.path, key) result = self.children_by_key.pop(key, *args, **kwargs) self.on_change() return result @@ -93,6 +96,9 @@ class DictMutableKeysEntity(EndpointEntity): child_obj.set(value) def change_key(self, old_key, new_key): + if old_key in self.required_keys: + raise RequiredKeyModified(self.path, old_key) + if new_key == old_key: return self.children_by_key[new_key] = self.children_by_key.pop(old_key) @@ -309,6 +315,10 @@ class DictMutableKeysEntity(EndpointEntity): for key in tuple(self.children_by_key.keys()): self.children_by_key.pop(key) + for required_key in self.required_keys: + if required_key not in new_value: + new_value[required_key] = NOT_SET + # Create new children children_label_by_id = {} metadata_labels = metadata.get(M_DYNAMIC_KEY_LABEL) or {} @@ -441,7 +451,13 @@ class DictMutableKeysEntity(EndpointEntity): def update_default_value(self, value): value = self._check_update_value(value, "default") - self.has_default_value = value is not NOT_SET + has_default_value = value is not NOT_SET + if has_default_value: + for required_key in self.required_keys: + if required_key not in value: + has_default_value = False + break + self.has_default_value = has_default_value value, metadata = self._prepare_value(value) self._default_value = value self._default_metadata = metadata diff --git a/pype/settings/entities/exceptions.py b/pype/settings/entities/exceptions.py index 951cd07243..7080a9b187 100644 --- a/pype/settings/entities/exceptions.py +++ b/pype/settings/entities/exceptions.py @@ -28,7 +28,17 @@ class InvalidValueType(Exception): super(InvalidValueType, self).__init__(msg) -class SchemaMissingFileInfo(Exception): +class RequiredKeyModified(KeyError): + def __init__(self, entity_path, key): + msg = "{} - Tried to modify required key \"{}\"." + super(RequiredKeyModified, self).__init__(msg.format(entity_path, key)) + + +class SchemaError(Exception): + pass + + +class SchemaMissingFileInfo(SchemaError): def __init__(self, invalid): full_path_keys = [] for item in invalid: @@ -41,7 +51,7 @@ class SchemaMissingFileInfo(Exception): super(SchemaMissingFileInfo, self).__init__(msg) -class SchemeGroupHierarchyBug(Exception): +class SchemeGroupHierarchyBug(SchemaError): def __init__(self, entity_path): msg = ( "Items with attribute \"is_group\" can't have another item with" @@ -50,7 +60,7 @@ class SchemeGroupHierarchyBug(Exception): super(SchemeGroupHierarchyBug, self).__init__(msg) -class SchemaDuplicatedKeys(Exception): +class SchemaDuplicatedKeys(SchemaError): def __init__(self, entity_path, key): msg = ( "Schema item contain duplicated key \"{}\" in" @@ -59,7 +69,7 @@ class SchemaDuplicatedKeys(Exception): super(SchemaDuplicatedKeys, self).__init__(msg) -class SchemaDuplicatedEnvGroupKeys(Exception): +class SchemaDuplicatedEnvGroupKeys(SchemaError): def __init__(self, invalid): items = [] for key_path, keys in invalid.items(): @@ -74,7 +84,7 @@ class SchemaDuplicatedEnvGroupKeys(Exception): super(SchemaDuplicatedEnvGroupKeys, self).__init__(msg) -class SchemaTemplateMissingKeys(Exception): +class SchemaTemplateMissingKeys(SchemaError): def __init__(self, missing_keys, required_keys, template_name=None): self.missing_keys = missing_keys self.required_keys = required_keys diff --git a/pype/settings/entities/schemas/system_schema/example_schema.json b/pype/settings/entities/schemas/system_schema/example_schema.json index 6e7a47d1bf..48a21cc0c6 100644 --- a/pype/settings/entities/schemas/system_schema/example_schema.json +++ b/pype/settings/entities/schemas/system_schema/example_schema.json @@ -141,6 +141,16 @@ "maximum": 100 } }, + { + "type": "dict-modifiable", + "key": "modifiable_dict_with_required_keys", + "label": "Modifiable dict with required keys", + "required_keys": [ + "key_1", + "key_2" + ], + "object_type": "text" + }, { "type": "list-strict", "key": "strict_list_labels_horizontal", diff --git a/pype/tools/settings/settings/widgets/dict_mutable_widget.py b/pype/tools/settings/settings/widgets/dict_mutable_widget.py index b27e0e492b..0cb051082e 100644 --- a/pype/tools/settings/settings/widgets/dict_mutable_widget.py +++ b/pype/tools/settings/settings/widgets/dict_mutable_widget.py @@ -358,7 +358,8 @@ class ModifiableDictItem(QtWidgets.QWidget): self.add_btn.setEnabled(False) def set_as_last_required(self): - self.add_btn.setEnabled(True) + if not self.collapsible_key: + self.add_btn.setEnabled(True) def _on_focus_lose(self): if ( @@ -827,10 +828,25 @@ class DictMutableKeysWidget(BaseWidget): while self.input_fields: self.remove_row(self.input_fields[0]) - for key, child_entity in self.entity.items(): + keys_order = list(self.entity.required_keys) + last_required = None + if keys_order: + last_required = keys_order[-1] + for key in self.entity.keys(): + if key in keys_order: + continue + keys_order.append(key) + + for key in keys_order: + child_entity = self.entity[key] input_field = self.add_widget_for_child(child_entity) input_field.origin_key = key - input_field.set_key(key) + if key in self.entity.required_keys: + input_field.set_as_required(key) + if key == last_required: + input_field.set_as_last_required() + else: + input_field.set_key(key) if self.entity.collapsible_key: label = self.entity.get_child_label(child_entity) input_field.origin_key_label = label