- Check if user \"{}\" has permissions" + " to access the Custom attribute
" + ).format(_session.api_key) + }) + items.append({ + "type": "label", + "value": "- Check if the Custom attribute exist
" + }) + return { + "items": items, + "title": "Synchronization failed", + "success": False, + "message": "Synchronization failed" + } + + # Find all entities in project + all_project_entities = self.session.query( + self.entities_query.format(ft_project_id) + ).all() + + # Store entities by `id` and `parent_id` + entities_dict = collections.defaultdict(lambda: { + "children": list(), + "parent_id": None, + "entity": None, + "entity_type": None, + "name": None, + "custom_attributes": {}, + "hier_attrs": {}, + "avalon_attrs": {}, + "tasks": [] + }) + + for entity in all_project_entities: + parent_id = entity["parent_id"] + entity_type = entity.entity_type + entity_type_low = entity_type.lower() + if entity_type_low == "task": + entities_dict[parent_id]["tasks"].append(entity["name"]) + continue + + entity_id = entity["id"] + entities_dict[entity_id].update({ + "entity": entity, + "parent_id": parent_id, + "entity_type": entity_type_low, + "entity_type_orig": entity_type, + "name": entity["name"] + }) + entities_dict[parent_id]["children"].append(entity_id) + + entities_dict[ft_project_id]["entity"] = ft_project + entities_dict[ft_project_id]["entity_type"] = ( + ft_project.entity_type.lower() + ) + entities_dict[ft_project_id]["entity_type_orig"] = ( + ft_project.entity_type + ) + entities_dict[ft_project_id]["name"] = ft_project["full_name"] + + self.ft_project_id = ft_project_id + self.entities_dict = entities_dict + + @property + def avalon_ents_by_id(self): + if self._avalon_ents_by_id is None: + self._avalon_ents_by_id = {} + for entity in self.avalon_entities: + self._avalon_ents_by_id[str(entity["_id"])] = entity + + return self._avalon_ents_by_id + + @property + def avalon_ents_by_ftrack_id(self): + if self._avalon_ents_by_ftrack_id is None: + self._avalon_ents_by_ftrack_id = {} + for entity in self.avalon_entities: + key = entity.get("data", {}).get("ftrackId") + if not key: + continue + self._avalon_ents_by_ftrack_id[key] = str(entity["_id"]) + + return self._avalon_ents_by_ftrack_id + + @property + def avalon_ents_by_name(self): + if self._avalon_ents_by_name is None: + self._avalon_ents_by_name = {} + for entity in self.avalon_entities: + self._avalon_ents_by_name[entity["name"]] = str(entity["_id"]) + + return self._avalon_ents_by_name + + @property + def avalon_ents_by_parent_id(self): + if self._avalon_ents_by_parent_id is None: + self._avalon_ents_by_parent_id = collections.defaultdict(list) + for entity in self.avalon_entities: + parent_id = entity["data"]["visualParent"] + if parent_id is not None: + parent_id = str(parent_id) + self._avalon_ents_by_parent_id[parent_id].append(entity) + + return self._avalon_ents_by_parent_id + + @property + def avalon_archived_ents(self): + if self._avalon_archived_ents is None: + self._avalon_archived_ents = [ + ent for ent in self.dbcon.find({"type": "archived_asset"}) + ] + return self._avalon_archived_ents + + @property + def avalon_archived_by_name(self): + if self._avalon_archived_by_name is None: + self._avalon_archived_by_name = collections.defaultdict(list) + for ent in self.avalon_archived_ents: + self._avalon_archived_by_name[ent["name"]].append(ent) + return self._avalon_archived_by_name + + @property + def avalon_archived_by_id(self): + if self._avalon_archived_by_id is None: + self._avalon_archived_by_id = { + str(ent["_id"]): ent for ent in self.avalon_archived_ents + } + return self._avalon_archived_by_id + + @property + def avalon_archived_by_parent_id(self): + if self._avalon_archived_by_parent_id is None: + self._avalon_archived_by_parent_id = collections.defaultdict(list) + for entity in self.avalon_archived_ents: + parent_id = entity["data"]["visualParent"] + if parent_id is not None: + parent_id = str(parent_id) + self._avalon_archived_by_parent_id[parent_id].append(entity) + + return self._avalon_archived_by_parent_id + + @property + def subsets_by_parent_id(self): + if self._subsets_by_parent_id is None: + self._subsets_by_parent_id = collections.defaultdict(list) + for subset in self.dbcon.find({"type": "subset"}): + self._subsets_by_parent_id[str(subset["parent"])].append( + subset + ) + + return self._subsets_by_parent_id + + @property + def changeability_by_mongo_id(self): + if self._changeability_by_mongo_id is None: + self._changeability_by_mongo_id = collections.defaultdict( + lambda: True + ) + self._changeability_by_mongo_id[self.avalon_project_id] = False + self._bubble_changeability(list(self.subsets_by_parent_id.keys())) + return self._changeability_by_mongo_id + + @property + def all_ftrack_names(self): + return [ + ent_dict["name"] for ent_dict in self.entities_dict.values() if ( + ent_dict.get("name") + ) + ] + + def duplicity_regex_check(self): + self.log.debug("* Checking duplicities and invalid symbols") + # Duplicity and regex check + entity_ids_by_name = {} + duplicates = [] + failed_regex = [] + task_names = {} + for ftrack_id, entity_dict in self.entities_dict.items(): + regex_check = True + name = entity_dict["name"] + entity_type = entity_dict["entity_type"] + # Tasks must be checked too + for task_name in entity_dict["tasks"]: + passed = task_names.get(task_name) + if passed is None: + passed = self.check_regex(task_name, "task") + task_names[task_name] = passed + + if not passed: + self.tasks_failed_regex[task_name].append(ftrack_id) + + if name in entity_ids_by_name: + duplicates.append(name) + else: + entity_ids_by_name[name] = [] + regex_check = self.check_regex(name, entity_type) + + entity_ids_by_name[name].append(ftrack_id) + if not regex_check: + failed_regex.append(name) + + for name in failed_regex: + self.failed_regex[name] = entity_ids_by_name[name] + + for name in duplicates: + self.duplicates[name] = entity_ids_by_name[name] + + self.filter_by_duplicate_regex() + + def check_regex(self, name, entity_type, in_schema=None): + schema_name = "asset-3.0" + if in_schema: + schema_name = in_schema + elif entity_type == "project": + schema_name = "project-2.0" + elif entity_type == "task": + schema_name = "task" + + name_pattern = self.schema_patterns.get(schema_name) + if not name_pattern: + default_pattern = "^[a-zA-Z0-9_.]*$" + schema_obj = avalon.schema._cache.get(schema_name + ".json") + if not schema_obj: + name_pattern = default_pattern + else: + name_pattern = schema_obj.get( + "properties", {}).get( + "name", {}).get( + "pattern", default_pattern + ) + self.schema_patterns[schema_name] = name_pattern + + if re.match(name_pattern, name): + return True + return False + + def filter_by_duplicate_regex(self): + filter_queue = queue.Queue() + failed_regex_msg = "{} - Entity has invalid symbol/s in name" + duplicate_msg = "Multiple entities have name \"{}\":" + + for ids in self.failed_regex.values(): + for id in ids: + ent_path = self.get_ent_path(id) + self.log.warning(failed_regex_msg.format(ent_path)) + filter_queue.put(id) + + for name, ids in self.duplicates.items(): + self.log.warning(duplicate_msg.format(name)) + for id in ids: + ent_path = self.get_ent_path(id) + self.log.warning(ent_path) + filter_queue.put(id) + + filtered_ids = [] + while not filter_queue.empty(): + ftrack_id = filter_queue.get() + if ftrack_id in filtered_ids: + continue + + entity_dict = self.entities_dict.pop(ftrack_id, {}) + if not entity_dict: + continue + + self.all_filtered_entities[ftrack_id] = entity_dict + parent_id = entity_dict.get("parent_id") + if parent_id and parent_id in self.entities_dict: + if ftrack_id in self.entities_dict[parent_id]["children"]: + self.entities_dict[parent_id]["children"].remove(ftrack_id) + + filtered_ids.append(ftrack_id) + for child_id in entity_dict.get("children", []): + filter_queue.put(child_id) + + # self.all_filtered_ids.extend(filtered_ids) + + for name, ids in self.tasks_failed_regex.items(): + for id in ids: + if id not in self.entities_dict: + continue + self.entities_dict[id]["tasks"].remove(name) + ent_path = self.get_ent_path(id) + self.log.warning(failed_regex_msg.format( + "/".join([ent_path, name]) + )) + + def filter_by_ignore_sync(self): + # skip filtering if `ignore_sync` attribute do not exist + if self.entities_dict[self.ft_project_id]["avalon_attrs"].get( + self.ignore_custom_attr_key, "_notset_" + ) == "_notset_": + return + + self.filter_queue = queue.Queue() + self.filter_queue.put((self.ft_project_id, False)) + while not self.filter_queue.empty(): + parent_id, remove = self.filter_queue.get() + if remove: + parent_dict = self.entities_dict.pop(parent_id, {}) + self.all_filtered_entities[parent_id] = parent_dict + self.filtered_ids.append(parent_id) + else: + parent_dict = self.entities_dict.get(parent_id, {}) + + for child_id in parent_dict.get("children", []): + # keep original `remove` value for all childs + _remove = (remove is True) + if not _remove: + if self.entities_dict[child_id]["avalon_attrs"].get( + self.ignore_custom_attr_key + ): + self.entities_dict[parent_id]["children"].remove( + child_id + ) + _remove = True + self.filter_queue.put((child_id, _remove)) + + # self.all_filtered_ids.extend(self.filtered_ids) + + def filter_by_selection(self, event): + # BUGGY!!!! cause that entities are in deleted list + # TODO may be working when filtering happen after preparations + # - But this part probably does not have any functional reason + # - Time of synchronization probably won't be changed much + selected_ids = [] + for entity in event["data"]["selection"]: + # Skip if project is in selection + if entity["entityType"] == "show": + return + selected_ids.append(entity["entityId"]) + + sync_ids = [self.ft_project_id] + parents_queue = queue.Queue() + children_queue = queue.Queue() + for id in selected_ids: + # skip if already filtered with ignore sync custom attribute + if id in self.filtered_ids: + continue + + parents_queue.put(id) + children_queue.put(id) + + while not parents_queue.empty(): + id = parents_queue.get() + while True: + # Stops when parent is in sync_ids + if id in self.filtered_ids or id in sync_ids or id is None: + break + sync_ids.append(id) + id = self.entities_dict[id]["parent_id"] + + while not children_queue.empty(): + parent_id = children_queue.get() + for child_id in self.entities_dict[parent_id]["children"]: + if child_id in sync_ids or child_id in self.filtered_ids: + continue + sync_ids.append(child_id) + children_queue.put(child_id) + + # separate not selected and to process entities + for key, value in self.entities_dict.items(): + if key not in sync_ids: + self.not_selected_ids.append(key) + + for id in self.not_selected_ids: + # pop from entities + value = self.entities_dict.pop(id) + # remove entity from parent's children + parent_id = value["parent_id"] + if parent_id not in sync_ids: + continue + + self.entities_dict[parent_id]["children"].remove(id) + + def set_cutom_attributes(self): + self.log.debug("* Preparing custom attributes") + # Get custom attributes and values + custom_attrs, hier_attrs = self.get_avalon_attr(True) + ent_types = self.session.query("select id, name from ObjectType").all() + ent_types_by_name = { + ent_type["name"]: ent_type["id"] for ent_type in ent_types + } + + attrs = set() + # store default values per entity type + attrs_per_entity_type = collections.defaultdict(dict) + avalon_attrs = collections.defaultdict(dict) + # store also custom attribute configuration id for future use (create) + attrs_per_entity_type_ca_id = collections.defaultdict(dict) + avalon_attrs_ca_id = collections.defaultdict(dict) + + for cust_attr in custom_attrs: + key = cust_attr["key"] + attrs.add(key) + ca_ent_type = cust_attr["entity_type"] + if key.startswith("avalon_"): + if ca_ent_type == "show": + avalon_attrs[ca_ent_type][key] = cust_attr["default"] + avalon_attrs_ca_id[ca_ent_type][key] = cust_attr["id"] + else: + obj_id = cust_attr["object_type_id"] + avalon_attrs[obj_id][key] = cust_attr["default"] + avalon_attrs_ca_id[obj_id][key] = cust_attr["id"] + continue + + if ca_ent_type == "show": + attrs_per_entity_type[ca_ent_type][key] = cust_attr["default"] + attrs_per_entity_type_ca_id[ca_ent_type][key] = cust_attr["id"] + else: + obj_id = cust_attr["object_type_id"] + attrs_per_entity_type[obj_id][key] = cust_attr["default"] + attrs_per_entity_type_ca_id[obj_id][key] = cust_attr["id"] + + obj_id_ent_type_map = {} + sync_ids = [] + for entity_id, entity_dict in self.entities_dict.items(): + sync_ids.append(entity_id) + entity_type = entity_dict["entity_type"] + entity_type_orig = entity_dict["entity_type_orig"] + + if entity_type == "project": + attr_key = "show" + else: + map_key = obj_id_ent_type_map.get(entity_type_orig) + if not map_key: + # Put space between capitals + # (e.g. 'AssetBuild' -> 'Asset Build') + map_key = re.sub( + r"(\w)([A-Z])", r"\1 \2", entity_type_orig + ) + obj_id_ent_type_map[entity_type_orig] = map_key + + # Get object id of entity type + attr_key = ent_types_by_name.get(map_key) + + # Backup soluction when id is not found by prequeried objects + if not attr_key: + query = "ObjectType where name is \"{}\"".format(map_key) + attr_key = self.session.query(query).one()["id"] + ent_types_by_name[map_key] = attr_key + + prepared_attrs = attrs_per_entity_type.get(attr_key) + prepared_avalon_attr = avalon_attrs.get(attr_key) + prepared_attrs_ca_id = attrs_per_entity_type_ca_id.get(attr_key) + prepared_avalon_attr_ca_id = avalon_attrs_ca_id.get(attr_key) + if prepared_attrs: + self.entities_dict[entity_id]["custom_attributes"] = ( + prepared_attrs.copy() + ) + if prepared_attrs_ca_id: + self.entities_dict[entity_id]["custom_attributes_id"] = ( + prepared_attrs_ca_id.copy() + ) + if prepared_avalon_attr: + self.entities_dict[entity_id]["avalon_attrs"] = ( + prepared_avalon_attr.copy() + ) + if prepared_avalon_attr_ca_id: + self.entities_dict[entity_id]["avalon_attrs_id"] = ( + prepared_avalon_attr_ca_id.copy() + ) + + # TODO query custom attributes by entity_id + entity_ids_joined = ", ".join([ + "\"{}\"".format(id) for id in sync_ids + ]) + attributes_joined = ", ".join([ + "\"{}\"".format(name) for name in attrs + ]) + + cust_attr_query = ( + "select value, entity_id from CustomAttributeValue " + "where entity_id in ({}) and configuration.key in ({})" + ) + [values] = self.session._call([{ + "action": "query", + "expression": cust_attr_query.format( + entity_ids_joined, attributes_joined + ) + }]) + + for value in values["data"]: + entity_id = value["entity_id"] + key = value["configuration"]["key"] + store_key = "custom_attributes" + if key.startswith("avalon_"): + store_key = "avalon_attrs" + self.entities_dict[entity_id][store_key][key] = value["value"] + + # process hierarchical attributes + self.set_hierarchical_attribute(hier_attrs, sync_ids) + + def set_hierarchical_attribute(self, hier_attrs, sync_ids): + # collect all hierarchical attribute keys + # and prepare default values to project + attribute_names = [] + for attr in hier_attrs: + key = attr["key"] + attribute_names.append(key) + + store_key = "hier_attrs" + if key.startswith("avalon_"): + store_key = "avalon_attrs" + + self.entities_dict[self.ft_project_id][store_key][key] = ( + attr["default"] + ) + + # Prepare dict with all hier keys and None values + prepare_dict = {} + prepare_dict_avalon = {} + for attr in attribute_names: + if attr.startswith("avalon_"): + prepare_dict_avalon[attr] = None + else: + prepare_dict[attr] = None + + for id, entity_dict in self.entities_dict.items(): + # Skip project because has stored defaults at the moment + if entity_dict["entity_type"] == "project": + continue + entity_dict["hier_attrs"] = prepare_dict.copy() + for key, val in prepare_dict_avalon.items(): + entity_dict["avalon_attrs"][key] = val + + # Prepare values to query + entity_ids_joined = ", ".join([ + "\"{}\"".format(id) for id in sync_ids + ]) + attributes_joined = ", ".join([ + "\"{}\"".format(name) for name in attribute_names + ]) + [values] = self.session._call([{ + "action": "query", + "expression": ( + "select value, entity_id from CustomAttributeValue " + "where entity_id in ({}) and configuration.key in ({})" + ).format(entity_ids_joined, attributes_joined) + }]) + + avalon_hier = [] + for value in values["data"]: + if value["value"] is None: + continue + entity_id = value["entity_id"] + key = value["configuration"]["key"] + store_key = "hier_attrs" + if key.startswith("avalon_"): + store_key = "avalon_attrs" + avalon_hier.append(key) + self.entities_dict[entity_id][store_key][key] = value["value"] + + # Get dictionary with not None hierarchical values to pull to childs + top_id = self.ft_project_id + project_values = {} + for key, value in self.entities_dict[top_id]["hier_attrs"].items(): + if value is not None: + project_values[key] = value + + for key in avalon_hier: + value = self.entities_dict[top_id]["avalon_attrs"][key] + if value is not None: + project_values[key] = value + + hier_down_queue = queue.Queue() + hier_down_queue.put((project_values, top_id)) + + while not hier_down_queue.empty(): + hier_values, parent_id = hier_down_queue.get() + for child_id in self.entities_dict[parent_id]["children"]: + _hier_values = hier_values.copy() + for name in attribute_names: + store_key = "hier_attrs" + if name.startswith("avalon_"): + store_key = "avalon_attrs" + value = self.entities_dict[child_id][store_key][name] + if value is not None: + _hier_values[name] = value + + self.entities_dict[child_id]["hier_attrs"].update(_hier_values) + hier_down_queue.put((_hier_values, child_id)) + + def remove_from_archived(self, mongo_id): + entity = self.avalon_archived_by_id.pop(mongo_id, None) + if not entity: + return + + if self._avalon_archived_ents is not None: + if entity in self._avalon_archived_ents: + self._avalon_archived_ents.remove(entity) + + if self._avalon_archived_by_name is not None: + name = entity["name"] + if name in self._avalon_archived_by_name: + name_ents = self._avalon_archived_by_name[name] + if entity in name_ents: + if len(name_ents) == 1: + self._avalon_archived_by_name.pop(name) + else: + self._avalon_archived_by_name[name].remove(entity) + + # TODO use custom None instead of __NOTSET__ + if self._avalon_archived_by_parent_id is not None: + parent_id = entity.get("data", {}).get( + "visualParent", "__NOTSET__" + ) + if parent_id is not None: + parent_id = str(parent_id) + + if parent_id in self._avalon_archived_by_parent_id: + parent_list = self._avalon_archived_by_parent_id[parent_id] + if entity not in parent_list: + self._avalon_archived_by_parent_id[parent_id].remove( + entity + ) + + def prepare_ftrack_ent_data(self): + not_set_ids = [] + for id, entity_dict in self.entities_dict.items(): + entity = entity_dict["entity"] + if entity is None: + not_set_ids.append(id) + continue + + self.entities_dict[id]["final_entity"] = {} + self.entities_dict[id]["final_entity"]["name"] = ( + entity_dict["name"] + ) + data = {} + data["ftrackId"] = entity["id"] + data["entityType"] = entity_dict["entity_type_orig"] + + for key, val in entity_dict.get("custom_attributes", []).items(): + data[key] = val + + for key, val in entity_dict.get("hier_attrs", []).items(): + data[key] = val + + if id == self.ft_project_id: + data["code"] = entity["name"] + self.entities_dict[id]["final_entity"]["data"] = data + self.entities_dict[id]["final_entity"]["type"] = "project" + + proj_schema = entity["project_schema"] + task_types = proj_schema["_task_type_schema"]["types"] + self.entities_dict[id]["final_entity"]["config"] = { + "tasks": [{"name": tt["name"]} for tt in task_types], + "apps": self.get_project_apps(data) + } + continue + + ent_path_items = [ent["name"] for ent in entity["link"]] + parents = ent_path_items[1:len(ent_path_items)-1:] + hierarchy = "" + if len(parents) > 0: + hierarchy = os.path.sep.join(parents) + + data["parents"] = parents + data["hierarchy"] = hierarchy + data["tasks"] = self.entities_dict[id].pop("tasks", []) + self.entities_dict[id]["final_entity"]["data"] = data + self.entities_dict[id]["final_entity"]["type"] = "asset" + + if not_set_ids: + self.log.debug(( + "- Debug information: Filtering bug, in entities dict are " + "empty dicts (function should not affect) <{}>" + ).format("| ".join(not_set_ids))) + for id in not_set_ids: + self.entities_dict.pop(id) + + def get_project_apps(self, proj_data): + apps = [] + missing_toml_msg = "Missing config file for application" + error_msg = ( + "Unexpected error happend during preparation of application" + ) + for app in proj_data.get("applications"): + try: + toml_path = avalon.lib.which_app(app) + # TODO report + if not toml_path: + self.log.warning(missing_toml_msg + '"{}"'.format(app)) + self.report_items["warning"][missing_toml_msg].append(app) + continue + + apps.append({ + "name": app, + "label": toml.load(toml_path)["label"] + }) + except Exception: + # TODO report + self.report_items["warning"][error_msg].append(app) + self.log.warning(( + "Error has happened during preparing application \"{}\"" + ).format(app), exc_info=True) + return apps + + def get_ent_path(self, ftrack_id): + ent_path = self._ent_pats_by_ftrack_id.get(ftrack_id) + if not ent_path: + entity = self.entities_dict[ftrack_id]["entity"] + ent_path = "/".join( + [ent["name"] for ent in entity["link"]] + ) + self._ent_pats_by_ftrack_id[ftrack_id] = ent_path + + return ent_path + + def prepare_avalon_entities(self, ft_project_name): + self.log.debug(( + "* Preparing avalon entities " + "(separate to Create, Update and Deleted groups)" + )) + # Avalon entities + self.dbcon.install() + self.dbcon.Session["AVALON_PROJECT"] = ft_project_name + avalon_project = self.dbcon.find_one({"type": "project"}) + avalon_entities = self.dbcon.find({"type": "asset"}) + self.avalon_project = avalon_project + self.avalon_entities = avalon_entities + + ftrack_avalon_mapper = {} + avalon_ftrack_mapper = {} + create_ftrack_ids = [] + update_ftrack_ids = [] + + same_mongo_id = [] + all_mongo_ids = {} + for ftrack_id, entity_dict in self.entities_dict.items(): + mongo_id = entity_dict["avalon_attrs"].get(self.id_cust_attr) + if not mongo_id: + continue + if mongo_id in all_mongo_ids: + same_mongo_id.append(mongo_id) + else: + all_mongo_ids[mongo_id] = [] + all_mongo_ids[mongo_id].append(ftrack_id) + + if avalon_project: + mongo_id = str(avalon_project["_id"]) + ftrack_avalon_mapper[self.ft_project_id] = mongo_id + avalon_ftrack_mapper[mongo_id] = self.ft_project_id + update_ftrack_ids.append(self.ft_project_id) + else: + create_ftrack_ids.append(self.ft_project_id) + + # make it go hierarchically + prepare_queue = queue.Queue() + + for child_id in self.entities_dict[self.ft_project_id]["children"]: + prepare_queue.put(child_id) + + while not prepare_queue.empty(): + ftrack_id = prepare_queue.get() + for child_id in self.entities_dict[ftrack_id]["children"]: + prepare_queue.put(child_id) + + entity_dict = self.entities_dict[ftrack_id] + ent_path = self.get_ent_path(ftrack_id) + + mongo_id = entity_dict["avalon_attrs"].get(self.id_cust_attr) + av_ent_by_mongo_id = self.avalon_ents_by_id.get(mongo_id) + if av_ent_by_mongo_id: + av_ent_ftrack_id = av_ent_by_mongo_id.get("data", {}).get( + "ftrackId" + ) + is_right = False + else_match_better = False + if av_ent_ftrack_id and av_ent_ftrack_id == ftrack_id: + is_right = True + + elif mongo_id not in same_mongo_id: + is_right = True + + else: + ftrack_ids_with_same_mongo = all_mongo_ids[mongo_id] + for _ftrack_id in ftrack_ids_with_same_mongo: + if _ftrack_id == av_ent_ftrack_id: + continue + + _entity_dict = self.entities_dict[_ftrack_id] + _mongo_id = _entity_dict["avalon_attrs"][ + self.id_cust_attr + ] + _av_ent_by_mongo_id = self.avalon_ents_by_id.get( + _mongo_id + ) + _av_ent_ftrack_id = _av_ent_by_mongo_id.get( + "data", {} + ).get("ftrackId") + if _av_ent_ftrack_id == ftrack_id: + else_match_better = True + break + + if not is_right and not else_match_better: + entity = entity_dict["entity"] + ent_path_items = [ent["name"] for ent in entity["link"]] + parents = ent_path_items[1:len(ent_path_items)-1:] + av_parents = av_ent_by_mongo_id["data"]["parents"] + if av_parents == parents: + is_right = True + else: + name = entity_dict["name"] + av_name = av_ent_by_mongo_id["name"] + if name == av_name: + is_right = True + + if is_right: + self.log.debug( + "Existing (by MongoID) <{}>".format(ent_path) + ) + ftrack_avalon_mapper[ftrack_id] = mongo_id + avalon_ftrack_mapper[mongo_id] = ftrack_id + update_ftrack_ids.append(ftrack_id) + continue + + mongo_id = self.avalon_ents_by_ftrack_id.get(ftrack_id) + if not mongo_id: + mongo_id = self.avalon_ents_by_name.get(entity_dict["name"]) + if mongo_id: + self.log.debug( + "Existing (by matching name) <{}>".format(ent_path) + ) + else: + self.log.debug( + "Existing (by FtrackID in mongo) <{}>".format(ent_path) + ) + + if mongo_id: + ftrack_avalon_mapper[ftrack_id] = mongo_id + avalon_ftrack_mapper[mongo_id] = ftrack_id + update_ftrack_ids.append(ftrack_id) + continue + + self.log.debug("New <{}>".format(ent_path)) + create_ftrack_ids.append(ftrack_id) + + deleted_entities = [] + for mongo_id in self.avalon_ents_by_id: + if mongo_id in avalon_ftrack_mapper: + continue + deleted_entities.append(mongo_id) + + av_ent = self.avalon_ents_by_id[mongo_id] + av_ent_path_items = [p for p in av_ent["data"]["parents"]] + av_ent_path_items.append(av_ent["name"]) + self.log.debug("Deleted <{}>".format("/".join(av_ent_path_items))) + + self.ftrack_avalon_mapper = ftrack_avalon_mapper + self.avalon_ftrack_mapper = avalon_ftrack_mapper + self.create_ftrack_ids = create_ftrack_ids + self.update_ftrack_ids = update_ftrack_ids + self.deleted_entities = deleted_entities + + self.log.debug(( + "Ftrack -> Avalon comparation: New <{}> " + "| Existing <{}> | Deleted <{}>" + ).format( + len(create_ftrack_ids), + len(update_ftrack_ids), + len(deleted_entities) + )) + + def filter_with_children(self, ftrack_id): + if ftrack_id not in self.entities_dict: + return + ent_dict = self.entities_dict[ftrack_id] + parent_id = ent_dict["parent_id"] + self.entities_dict[parent_id]["children"].remove(ftrack_id) + + children_queue = queue.Queue() + children_queue.put(ftrack_id) + while not children_queue.empty(): + _ftrack_id = children_queue.get() + entity_dict = self.entities_dict.pop(_ftrack_id, {"children": []}) + for child_id in entity_dict["children"]: + children_queue.put(child_id) + + def prepare_changes(self): + self.log.debug("* Preparing changes for avalon/ftrack") + hierarchy_changing_ids = [] + ignore_keys = collections.defaultdict(list) + + update_queue = queue.Queue() + for ftrack_id in self.update_ftrack_ids: + update_queue.put(ftrack_id) + + while not update_queue.empty(): + ftrack_id = update_queue.get() + if ftrack_id == self.ft_project_id: + changes = self.prepare_project_changes() + if changes: + self.updates[self.avalon_project_id] = changes + continue + + ftrack_ent_dict = self.entities_dict[ftrack_id] + + # *** check parents + parent_check = False + + ftrack_parent_id = ftrack_ent_dict["parent_id"] + avalon_id = self.ftrack_avalon_mapper[ftrack_id] + avalon_entity = self.avalon_ents_by_id[avalon_id] + avalon_parent_id = avalon_entity["data"]["visualParent"] + if avalon_parent_id is not None: + avalon_parent_id = str(avalon_parent_id) + + ftrack_parent_mongo_id = self.ftrack_avalon_mapper[ + ftrack_parent_id + ] + + # if parent is project + if (ftrack_parent_mongo_id == avalon_parent_id) or ( + ftrack_parent_id == self.ft_project_id and + avalon_parent_id is None + ): + parent_check = True + + # check name + ftrack_name = ftrack_ent_dict["name"] + avalon_name = avalon_entity["name"] + name_check = ftrack_name == avalon_name + + # IDEAL STATE: both parent and name check passed + if parent_check and name_check: + continue + + # If entity is changeable then change values of parent or name + if self.changeability_by_mongo_id[avalon_id]: + # TODO logging + if not parent_check: + if ftrack_parent_mongo_id == str(self.avalon_project_id): + new_parent_name = self.entities_dict[ + self.ft_project_id]["name"] + new_parent_id = None + else: + new_parent_name = self.avalon_ents_by_id[ + ftrack_parent_mongo_id]["name"] + new_parent_id = ObjectId(ftrack_parent_mongo_id) + + if avalon_parent_id == str(self.avalon_project_id): + old_parent_name = self.entities_dict[ + self.ft_project_id]["name"] + else: + old_parent_name = self.avalon_ents_by_id[ + ftrack_parent_mongo_id]["name"] + + self.updates[avalon_id]["data"] = { + "visualParent": new_parent_id + } + ignore_keys[ftrack_id].append("data.visualParent") + self.log.debug(( + "Avalon entity \"{}\" changed parent \"{}\" -> \"{}\"" + ).format(avalon_name, old_parent_name, new_parent_name)) + + if not name_check: + self.updates[avalon_id]["name"] = ftrack_name + ignore_keys[ftrack_id].append("name") + self.log.debug( + "Avalon entity \"{}\" was renamed to \"{}\"".format( + avalon_name, ftrack_name + ) + ) + continue + + # parents and hierarchy must be recalculated + hierarchy_changing_ids.append(ftrack_id) + + # Parent is project if avalon_parent_id is set to None + if avalon_parent_id is None: + avalon_parent_id = str(self.avalon_project_id) + + if not name_check: + ent_path = self.get_ent_path(ftrack_id) + # TODO report + # TODO logging + self.entities_dict[ftrack_id]["name"] = avalon_name + self.entities_dict[ftrack_id]["entity"]["name"] = ( + avalon_name + ) + self.entities_dict[ftrack_id]["final_entity"]["name"] = ( + avalon_name + ) + self.log.warning("Name was changed back to {} <{}>".format( + avalon_name, ent_path + )) + self._ent_pats_by_ftrack_id.pop(ftrack_id, None) + msg = ( + "NOTE: Allowed symbols are Letters( a-Z )," + " Numbers( 0-9 ) and Underscore( _ )
" + ) + }) + log_msgs = [] + for name, ids in self.failed_regex.items(): + error_title = { + "type": "label", + "value": "## {}".format(name) + } + items.append(error_title) + paths = [] + for entity_id in ids: + ent_path = self.get_ent_path(entity_id) + paths.append(ent_path) + + error_message = { + "type": "label", + "value": '{}
'.format("{}
'.format("NOTE: It is not allowed to have same name" + " for multiple entities in one project
" + ) + }) + log_msgs = [] + for name, ids in self.duplicates.items(): + error_title = { + "type": "label", + "value": "## {}".format(name) + } + items.append(error_title) + paths = [] + for entity_id in ids: + ent_path = self.get_ent_path(entity_id) + paths.append(ent_path) + + error_message = { + "type": "label", + "value": '{}
'.format("{}
'.format("{}
".format( + str(traceback.format_exc()).replace( + "\n", "- Check if user \"{}\" has permissions" + " to access the Custom attribute
" + ).format(_session.api_key) + }) + items.append({ + "type": "label", + "value": "- Check if the Custom attribute exist
" }) - }) - session.commit() - try: - self.log.debug("Preparing entities for synchronization") - - if entities[0].entity_type.lower() == "project": - ft_project_name = entities[0]["full_name"] - else: - ft_project_name = entities[0]["project"]["full_name"] - - project_entities = session.query( - self.entities_query.format(ft_project_name) - ).all() - - ft_project = session.query( - self.project_query.format(ft_project_name) - ).one() - - entities_by_id = {} - entities_by_parent = collections.defaultdict(list) - - entities_by_id[ft_project["id"]] = ft_project - for ent in project_entities: - entities_by_id[ent["id"]] = ent - entities_by_parent[ent["parent_id"]].append(ent) - - importable = [] - for ent_info in event["data"]["selection"]: - ent = entities_by_id[ent_info["entityId"]] - for link_ent_info in ent["link"]: - link_ent = entities_by_id[link_ent_info["id"]] - if ( - ent.entity_type.lower() in self.ignore_entity_types or - link_ent in importable - ): - continue - - importable.append(link_ent) - - def add_children(parent_id): - ents = entities_by_parent[parent_id] - for ent in ents: - if ent.entity_type.lower() in self.ignore_entity_types: - continue - - if ent not in importable: - importable.append(ent) - - add_children(ent["id"]) - - # add children of selection to importable - for ent_info in event["data"]["selection"]: - add_children(ent_info["entityId"]) - - # Check names: REGEX in schema/duplicates - raise error if found - all_names = [] - duplicates = [] - - for entity in importable: - lib.avalon_check_name(entity) - if entity.entity_type.lower() == "project": - continue - - if entity['name'] in all_names: - duplicates.append("'{}'".format(entity['name'])) - else: - all_names.append(entity['name']) - - if len(duplicates) > 0: - # TODO Show information to user and return False - raise ValueError( - "Entity name duplication: {}".format(", ".join(duplicates)) - ) - - # ----- PROJECT ------ - avalon_project = lib.get_avalon_project(ft_project) - custom_attributes = lib.get_avalon_attr(session) - - # Import all entities to Avalon DB - for entity in importable: - result = lib.import_to_avalon( - session=session, - entity=entity, - ft_project=ft_project, - av_project=avalon_project, - custom_attributes=custom_attributes - ) - # TODO better error handling - # maybe split into critical, warnings and messages? - if 'errors' in result and len(result['errors']) > 0: - job['status'] = 'failed' - session.commit() - - lib.show_errors(self, event, result['errors']) - - return { - 'success': False, - 'message': "Sync to avalon FAILED" - } - - if avalon_project is None: - if 'project' in result: - avalon_project = result['project'] - - job['status'] = 'done' - session.commit() - - except ValueError as ve: - # TODO remove this part!!!! - job['status'] = 'failed' - session.commit() - message = str(ve) - self.log.error( - 'Error during syncToAvalon: {}'.format(message), - exc_info=True - ) - - except Exception as e: - job['status'] = 'failed' - session.commit() - exc_type, exc_obj, exc_tb = sys.exc_info() - fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] - log_message = "{}/{}/Line: {}".format( - exc_type, fname, exc_tb.tb_lineno - ) - self.log.error( - 'Error during syncToAvalon: {}'.format(log_message), - exc_info=True - ) - # TODO add traceback to message and show to user - message = ( - 'Unexpected Error' - ' - Please check Log for more information' - ) - - finally: - if job['status'] in ['queued', 'running']: - job['status'] = 'failed' - - session.commit() - - time_end = time.time() - self.log.debug("Synchronization took \"{}\"".format( - str(time_end - time_start) - )) - - if job["status"] != "failed": - self.log.debug("Triggering Sync hierarchical attributes") - self.trigger_action("sync.hierarchical.attrs", event) - - if len(message) > 0: - message = "Unable to sync: {}".format(message) return { - 'success': False, - 'message': message + "items": items, + "title": "Synchronization failed", + "success": False, + "message": "Synchronization failed" } + # Find all entities in project + all_project_entities = self.session.query( + self.entities_query.format(ft_project_id) + ).all() + + # Store entities by `id` and `parent_id` + entities_dict = collections.defaultdict(lambda: { + "children": list(), + "parent_id": None, + "entity": None, + "entity_type": None, + "name": None, + "custom_attributes": {}, + "hier_attrs": {}, + "avalon_attrs": {}, + "tasks": [] + }) + + for entity in all_project_entities: + parent_id = entity["parent_id"] + entity_type = entity.entity_type + entity_type_low = entity_type.lower() + if entity_type_low == "task": + entities_dict[parent_id]["tasks"].append(entity["name"]) + continue + + entity_id = entity["id"] + entities_dict[entity_id].update({ + "entity": entity, + "parent_id": parent_id, + "entity_type": entity_type_low, + "entity_type_orig": entity_type, + "name": entity["name"] + }) + entities_dict[parent_id]["children"].append(entity_id) + + entities_dict[ft_project_id]["entity"] = ft_project + entities_dict[ft_project_id]["entity_type"] = ( + ft_project.entity_type.lower() + ) + entities_dict[ft_project_id]["entity_type_orig"] = ( + ft_project.entity_type + ) + entities_dict[ft_project_id]["name"] = ft_project["full_name"] + + self.ft_project_id = ft_project_id + self.entities_dict = entities_dict + + @property + def avalon_ents_by_id(self): + if self._avalon_ents_by_id is None: + self._avalon_ents_by_id = {} + for entity in self.avalon_entities: + self._avalon_ents_by_id[str(entity["_id"])] = entity + + return self._avalon_ents_by_id + + @property + def avalon_ents_by_ftrack_id(self): + if self._avalon_ents_by_ftrack_id is None: + self._avalon_ents_by_ftrack_id = {} + for entity in self.avalon_entities: + key = entity.get("data", {}).get("ftrackId") + if not key: + continue + self._avalon_ents_by_ftrack_id[key] = str(entity["_id"]) + + return self._avalon_ents_by_ftrack_id + + @property + def avalon_ents_by_name(self): + if self._avalon_ents_by_name is None: + self._avalon_ents_by_name = {} + for entity in self.avalon_entities: + self._avalon_ents_by_name[entity["name"]] = str(entity["_id"]) + + return self._avalon_ents_by_name + + @property + def avalon_ents_by_parent_id(self): + if self._avalon_ents_by_parent_id is None: + self._avalon_ents_by_parent_id = collections.defaultdict(list) + for entity in self.avalon_entities: + parent_id = entity["data"]["visualParent"] + if parent_id is not None: + parent_id = str(parent_id) + self._avalon_ents_by_parent_id[parent_id].append(entity) + + return self._avalon_ents_by_parent_id + + @property + def avalon_archived_ents(self): + if self._avalon_archived_ents is None: + self._avalon_archived_ents = [ + ent for ent in self.dbcon.find({"type": "archived_asset"}) + ] + return self._avalon_archived_ents + + @property + def avalon_archived_by_name(self): + if self._avalon_archived_by_name is None: + self._avalon_archived_by_name = collections.defaultdict(list) + for ent in self.avalon_archived_ents: + self._avalon_archived_by_name[ent["name"]].append(ent) + return self._avalon_archived_by_name + + @property + def avalon_archived_by_id(self): + if self._avalon_archived_by_id is None: + self._avalon_archived_by_id = { + str(ent["_id"]): ent for ent in self.avalon_archived_ents + } + return self._avalon_archived_by_id + + @property + def avalon_archived_by_parent_id(self): + if self._avalon_archived_by_parent_id is None: + self._avalon_archived_by_parent_id = collections.defaultdict(list) + for entity in self.avalon_archived_ents: + parent_id = entity["data"]["visualParent"] + if parent_id is not None: + parent_id = str(parent_id) + self._avalon_archived_by_parent_id[parent_id].append(entity) + + return self._avalon_archived_by_parent_id + + @property + def subsets_by_parent_id(self): + if self._subsets_by_parent_id is None: + self._subsets_by_parent_id = collections.defaultdict(list) + for subset in self.dbcon.find({"type": "subset"}): + self._subsets_by_parent_id[str(subset["parent"])].append( + subset + ) + + return self._subsets_by_parent_id + + @property + def changeability_by_mongo_id(self): + if self._changeability_by_mongo_id is None: + self._changeability_by_mongo_id = collections.defaultdict( + lambda: True + ) + self._changeability_by_mongo_id[self.avalon_project_id] = False + self._bubble_changeability(list(self.subsets_by_parent_id.keys())) + return self._changeability_by_mongo_id + + @property + def all_ftrack_names(self): + return [ + ent_dict["name"] for ent_dict in self.entities_dict.values() if ( + ent_dict.get("name") + ) + ] + + def duplicity_regex_check(self): + self.log.debug("* Checking duplicities and invalid symbols") + # Duplicity and regex check + entity_ids_by_name = {} + duplicates = [] + failed_regex = [] + task_names = {} + for ftrack_id, entity_dict in self.entities_dict.items(): + regex_check = True + name = entity_dict["name"] + entity_type = entity_dict["entity_type"] + # Tasks must be checked too + for task_name in entity_dict["tasks"]: + passed = task_names.get(task_name) + if passed is None: + passed = self.check_regex(task_name, "task") + task_names[task_name] = passed + + if not passed: + self.tasks_failed_regex[task_name].append(ftrack_id) + + if name in entity_ids_by_name: + duplicates.append(name) + else: + entity_ids_by_name[name] = [] + regex_check = self.check_regex(name, entity_type) + + entity_ids_by_name[name].append(ftrack_id) + if not regex_check: + failed_regex.append(name) + + for name in failed_regex: + self.failed_regex[name] = entity_ids_by_name[name] + + for name in duplicates: + self.duplicates[name] = entity_ids_by_name[name] + + self.filter_by_duplicate_regex() + + def check_regex(self, name, entity_type, in_schema=None): + schema_name = "asset-3.0" + if in_schema: + schema_name = in_schema + elif entity_type == "project": + schema_name = "project-2.0" + elif entity_type == "task": + schema_name = "task" + + name_pattern = self.schema_patterns.get(schema_name) + if not name_pattern: + default_pattern = "^[a-zA-Z0-9_.]*$" + schema_obj = avalon.schema._cache.get(schema_name + ".json") + if not schema_obj: + name_pattern = default_pattern + else: + name_pattern = schema_obj.get( + "properties", {}).get( + "name", {}).get( + "pattern", default_pattern + ) + self.schema_patterns[schema_name] = name_pattern + + if re.match(name_pattern, name): + return True + return False + + def filter_by_duplicate_regex(self): + filter_queue = queue.Queue() + failed_regex_msg = "{} - Entity has invalid symbol/s in name" + duplicate_msg = "Multiple entities have name \"{}\":" + + for ids in self.failed_regex.values(): + for id in ids: + ent_path = self.get_ent_path(id) + self.log.warning(failed_regex_msg.format(ent_path)) + filter_queue.put(id) + + for name, ids in self.duplicates.items(): + self.log.warning(duplicate_msg.format(name)) + for id in ids: + ent_path = self.get_ent_path(id) + self.log.warning(ent_path) + filter_queue.put(id) + + filtered_ids = [] + while not filter_queue.empty(): + ftrack_id = filter_queue.get() + if ftrack_id in filtered_ids: + continue + + entity_dict = self.entities_dict.pop(ftrack_id, {}) + if not entity_dict: + continue + + self.all_filtered_entities[ftrack_id] = entity_dict + parent_id = entity_dict.get("parent_id") + if parent_id and parent_id in self.entities_dict: + if ftrack_id in self.entities_dict[parent_id]["children"]: + self.entities_dict[parent_id]["children"].remove(ftrack_id) + + filtered_ids.append(ftrack_id) + for child_id in entity_dict.get("children", []): + filter_queue.put(child_id) + + # self.all_filtered_ids.extend(filtered_ids) + + for name, ids in self.tasks_failed_regex.items(): + for id in ids: + if id not in self.entities_dict: + continue + self.entities_dict[id]["tasks"].remove(name) + ent_path = self.get_ent_path(id) + self.log.warning(failed_regex_msg.format( + "/".join([ent_path, name]) + )) + + def filter_by_ignore_sync(self): + # skip filtering if `ignore_sync` attribute do not exist + if self.entities_dict[self.ft_project_id]["avalon_attrs"].get( + self.ignore_custom_attr_key, "_notset_" + ) == "_notset_": + return + + self.filter_queue = queue.Queue() + self.filter_queue.put((self.ft_project_id, False)) + while not self.filter_queue.empty(): + parent_id, remove = self.filter_queue.get() + if remove: + parent_dict = self.entities_dict.pop(parent_id, {}) + self.all_filtered_entities[parent_id] = parent_dict + self.filtered_ids.append(parent_id) + else: + parent_dict = self.entities_dict.get(parent_id, {}) + + for child_id in parent_dict.get("children", []): + # keep original `remove` value for all childs + _remove = (remove is True) + if not _remove: + if self.entities_dict[child_id]["avalon_attrs"].get( + self.ignore_custom_attr_key + ): + self.entities_dict[parent_id]["children"].remove( + child_id + ) + _remove = True + self.filter_queue.put((child_id, _remove)) + + # self.all_filtered_ids.extend(self.filtered_ids) + + def filter_by_selection(self, event): + # BUGGY!!!! cause that entities are in deleted list + # TODO may be working when filtering happen after preparations + # - But this part probably does not have any functional reason + # - Time of synchronization probably won't be changed much + selected_ids = [] + for entity in event["data"]["selection"]: + # Skip if project is in selection + if entity["entityType"] == "show": + return + selected_ids.append(entity["entityId"]) + + sync_ids = [self.ft_project_id] + parents_queue = queue.Queue() + children_queue = queue.Queue() + for id in selected_ids: + # skip if already filtered with ignore sync custom attribute + if id in self.filtered_ids: + continue + + parents_queue.put(id) + children_queue.put(id) + + while not parents_queue.empty(): + id = parents_queue.get() + while True: + # Stops when parent is in sync_ids + if id in self.filtered_ids or id in sync_ids or id is None: + break + sync_ids.append(id) + id = self.entities_dict[id]["parent_id"] + + while not children_queue.empty(): + parent_id = children_queue.get() + for child_id in self.entities_dict[parent_id]["children"]: + if child_id in sync_ids or child_id in self.filtered_ids: + continue + sync_ids.append(child_id) + children_queue.put(child_id) + + # separate not selected and to process entities + for key, value in self.entities_dict.items(): + if key not in sync_ids: + self.not_selected_ids.append(key) + + for id in self.not_selected_ids: + # pop from entities + value = self.entities_dict.pop(id) + # remove entity from parent's children + parent_id = value["parent_id"] + if parent_id not in sync_ids: + continue + + self.entities_dict[parent_id]["children"].remove(id) + + def set_cutom_attributes(self): + self.log.debug("* Preparing custom attributes") + # Get custom attributes and values + custom_attrs, hier_attrs = self.get_avalon_attr(True) + ent_types = self.session.query("select id, name from ObjectType").all() + ent_types_by_name = { + ent_type["name"]: ent_type["id"] for ent_type in ent_types + } + + attrs = set() + # store default values per entity type + attrs_per_entity_type = collections.defaultdict(dict) + avalon_attrs = collections.defaultdict(dict) + # store also custom attribute configuration id for future use (create) + attrs_per_entity_type_ca_id = collections.defaultdict(dict) + avalon_attrs_ca_id = collections.defaultdict(dict) + + for cust_attr in custom_attrs: + key = cust_attr["key"] + attrs.add(key) + ca_ent_type = cust_attr["entity_type"] + if key.startswith("avalon_"): + if ca_ent_type == "show": + avalon_attrs[ca_ent_type][key] = cust_attr["default"] + avalon_attrs_ca_id[ca_ent_type][key] = cust_attr["id"] + else: + obj_id = cust_attr["object_type_id"] + avalon_attrs[obj_id][key] = cust_attr["default"] + avalon_attrs_ca_id[obj_id][key] = cust_attr["id"] + continue + + if ca_ent_type == "show": + attrs_per_entity_type[ca_ent_type][key] = cust_attr["default"] + attrs_per_entity_type_ca_id[ca_ent_type][key] = cust_attr["id"] + else: + obj_id = cust_attr["object_type_id"] + attrs_per_entity_type[obj_id][key] = cust_attr["default"] + attrs_per_entity_type_ca_id[obj_id][key] = cust_attr["id"] + + obj_id_ent_type_map = {} + sync_ids = [] + for entity_id, entity_dict in self.entities_dict.items(): + sync_ids.append(entity_id) + entity_type = entity_dict["entity_type"] + entity_type_orig = entity_dict["entity_type_orig"] + + if entity_type == "project": + attr_key = "show" + else: + map_key = obj_id_ent_type_map.get(entity_type_orig) + if not map_key: + # Put space between capitals + # (e.g. 'AssetBuild' -> 'Asset Build') + map_key = re.sub( + r"(\w)([A-Z])", r"\1 \2", entity_type_orig + ) + obj_id_ent_type_map[entity_type_orig] = map_key + + # Get object id of entity type + attr_key = ent_types_by_name.get(map_key) + + # Backup soluction when id is not found by prequeried objects + if not attr_key: + query = "ObjectType where name is \"{}\"".format(map_key) + attr_key = self.session.query(query).one()["id"] + ent_types_by_name[map_key] = attr_key + + prepared_attrs = attrs_per_entity_type.get(attr_key) + prepared_avalon_attr = avalon_attrs.get(attr_key) + prepared_attrs_ca_id = attrs_per_entity_type_ca_id.get(attr_key) + prepared_avalon_attr_ca_id = avalon_attrs_ca_id.get(attr_key) + if prepared_attrs: + self.entities_dict[entity_id]["custom_attributes"] = ( + prepared_attrs.copy() + ) + if prepared_attrs_ca_id: + self.entities_dict[entity_id]["custom_attributes_id"] = ( + prepared_attrs_ca_id.copy() + ) + if prepared_avalon_attr: + self.entities_dict[entity_id]["avalon_attrs"] = ( + prepared_avalon_attr.copy() + ) + if prepared_avalon_attr_ca_id: + self.entities_dict[entity_id]["avalon_attrs_id"] = ( + prepared_avalon_attr_ca_id.copy() + ) + + # TODO query custom attributes by entity_id + entity_ids_joined = ", ".join([ + "\"{}\"".format(id) for id in sync_ids + ]) + attributes_joined = ", ".join([ + "\"{}\"".format(name) for name in attrs + ]) + + cust_attr_query = ( + "select value, entity_id from CustomAttributeValue " + "where entity_id in ({}) and configuration.key in ({})" + ) + [values] = self.session._call([{ + "action": "query", + "expression": cust_attr_query.format( + entity_ids_joined, attributes_joined + ) + }]) + + for value in values["data"]: + entity_id = value["entity_id"] + key = value["configuration"]["key"] + store_key = "custom_attributes" + if key.startswith("avalon_"): + store_key = "avalon_attrs" + self.entities_dict[entity_id][store_key][key] = value["value"] + + # process hierarchical attributes + self.set_hierarchical_attribute(hier_attrs, sync_ids) + + def set_hierarchical_attribute(self, hier_attrs, sync_ids): + # collect all hierarchical attribute keys + # and prepare default values to project + attribute_names = [] + for attr in hier_attrs: + key = attr["key"] + attribute_names.append(key) + + store_key = "hier_attrs" + if key.startswith("avalon_"): + store_key = "avalon_attrs" + + self.entities_dict[self.ft_project_id][store_key][key] = ( + attr["default"] + ) + + # Prepare dict with all hier keys and None values + prepare_dict = {} + prepare_dict_avalon = {} + for attr in attribute_names: + if attr.startswith("avalon_"): + prepare_dict_avalon[attr] = None + else: + prepare_dict[attr] = None + + for id, entity_dict in self.entities_dict.items(): + # Skip project because has stored defaults at the moment + if entity_dict["entity_type"] == "project": + continue + entity_dict["hier_attrs"] = prepare_dict.copy() + for key, val in prepare_dict_avalon.items(): + entity_dict["avalon_attrs"][key] = val + + # Prepare values to query + entity_ids_joined = ", ".join([ + "\"{}\"".format(id) for id in sync_ids + ]) + attributes_joined = ", ".join([ + "\"{}\"".format(name) for name in attribute_names + ]) + [values] = self.session._call([{ + "action": "query", + "expression": ( + "select value, entity_id from CustomAttributeValue " + "where entity_id in ({}) and configuration.key in ({})" + ).format(entity_ids_joined, attributes_joined) + }]) + + avalon_hier = [] + for value in values["data"]: + if value["value"] is None: + continue + entity_id = value["entity_id"] + key = value["configuration"]["key"] + store_key = "hier_attrs" + if key.startswith("avalon_"): + store_key = "avalon_attrs" + avalon_hier.append(key) + self.entities_dict[entity_id][store_key][key] = value["value"] + + # Get dictionary with not None hierarchical values to pull to childs + top_id = self.ft_project_id + project_values = {} + for key, value in self.entities_dict[top_id]["hier_attrs"].items(): + if value is not None: + project_values[key] = value + + for key in avalon_hier: + value = self.entities_dict[top_id]["avalon_attrs"][key] + if value is not None: + project_values[key] = value + + hier_down_queue = queue.Queue() + hier_down_queue.put((project_values, top_id)) + + while not hier_down_queue.empty(): + hier_values, parent_id = hier_down_queue.get() + for child_id in self.entities_dict[parent_id]["children"]: + _hier_values = hier_values.copy() + for name in attribute_names: + store_key = "hier_attrs" + if name.startswith("avalon_"): + store_key = "avalon_attrs" + value = self.entities_dict[child_id][store_key][name] + if value is not None: + _hier_values[name] = value + + self.entities_dict[child_id]["hier_attrs"].update(_hier_values) + hier_down_queue.put((_hier_values, child_id)) + + def remove_from_archived(self, mongo_id): + entity = self.avalon_archived_by_id.pop(mongo_id, None) + if not entity: + return + + if self._avalon_archived_ents is not None: + if entity in self._avalon_archived_ents: + self._avalon_archived_ents.remove(entity) + + if self._avalon_archived_by_name is not None: + name = entity["name"] + if name in self._avalon_archived_by_name: + name_ents = self._avalon_archived_by_name[name] + if entity in name_ents: + if len(name_ents) == 1: + self._avalon_archived_by_name.pop(name) + else: + self._avalon_archived_by_name[name].remove(entity) + + # TODO use custom None instead of __NOTSET__ + if self._avalon_archived_by_parent_id is not None: + parent_id = entity.get("data", {}).get( + "visualParent", "__NOTSET__" + ) + if parent_id is not None: + parent_id = str(parent_id) + + if parent_id in self._avalon_archived_by_parent_id: + parent_list = self._avalon_archived_by_parent_id[parent_id] + if entity not in parent_list: + self._avalon_archived_by_parent_id[parent_id].remove( + entity + ) + + def prepare_ftrack_ent_data(self): + not_set_ids = [] + for id, entity_dict in self.entities_dict.items(): + entity = entity_dict["entity"] + if entity is None: + not_set_ids.append(id) + continue + + self.entities_dict[id]["final_entity"] = {} + self.entities_dict[id]["final_entity"]["name"] = ( + entity_dict["name"] + ) + data = {} + data["ftrackId"] = entity["id"] + data["entityType"] = entity_dict["entity_type_orig"] + + for key, val in entity_dict.get("custom_attributes", []).items(): + data[key] = val + + for key, val in entity_dict.get("hier_attrs", []).items(): + data[key] = val + + if id == self.ft_project_id: + data["code"] = entity["name"] + self.entities_dict[id]["final_entity"]["data"] = data + self.entities_dict[id]["final_entity"]["type"] = "project" + + proj_schema = entity["project_schema"] + task_types = proj_schema["_task_type_schema"]["types"] + self.entities_dict[id]["final_entity"]["config"] = { + "tasks": [{"name": tt["name"]} for tt in task_types], + "apps": self.get_project_apps(data) + } + continue + + ent_path_items = [ent["name"] for ent in entity["link"]] + parents = ent_path_items[1:len(ent_path_items)-1:] + hierarchy = "" + if len(parents) > 0: + hierarchy = os.path.sep.join(parents) + + data["parents"] = parents + data["hierarchy"] = hierarchy + data["tasks"] = self.entities_dict[id].pop("tasks", []) + self.entities_dict[id]["final_entity"]["data"] = data + self.entities_dict[id]["final_entity"]["type"] = "asset" + + if not_set_ids: + self.log.debug(( + "- Debug information: Filtering bug, in entities dict are " + "empty dicts (function should not affect) <{}>" + ).format("| ".join(not_set_ids))) + for id in not_set_ids: + self.entities_dict.pop(id) + + def get_project_apps(self, proj_data): + apps = [] + missing_toml_msg = "Missing config file for application" + error_msg = ( + "Unexpected error happend during preparation of application" + ) + for app in proj_data.get("applications"): + try: + toml_path = avalon.lib.which_app(app) + # TODO report + if not toml_path: + self.log.warning(missing_toml_msg + '"{}"'.format(app)) + self.report_items["warning"][missing_toml_msg].append(app) + continue + + apps.append({ + "name": app, + "label": toml.load(toml_path)["label"] + }) + except Exception: + # TODO report + self.report_items["warning"][error_msg].append(app) + self.log.warning(( + "Error has happened during preparing application \"{}\"" + ).format(app), exc_info=True) + return apps + + def get_ent_path(self, ftrack_id): + ent_path = self._ent_pats_by_ftrack_id.get(ftrack_id) + if not ent_path: + entity = self.entities_dict[ftrack_id]["entity"] + ent_path = "/".join( + [ent["name"] for ent in entity["link"]] + ) + self._ent_pats_by_ftrack_id[ftrack_id] = ent_path + + return ent_path + + def prepare_avalon_entities(self, ft_project_name): + self.log.debug(( + "* Preparing avalon entities " + "(separate to Create, Update and Deleted groups)" + )) + # Avalon entities + self.dbcon.install() + self.dbcon.Session["AVALON_PROJECT"] = ft_project_name + avalon_project = self.dbcon.find_one({"type": "project"}) + avalon_entities = self.dbcon.find({"type": "asset"}) + self.avalon_project = avalon_project + self.avalon_entities = avalon_entities + + ftrack_avalon_mapper = {} + avalon_ftrack_mapper = {} + create_ftrack_ids = [] + update_ftrack_ids = [] + + same_mongo_id = [] + all_mongo_ids = {} + for ftrack_id, entity_dict in self.entities_dict.items(): + mongo_id = entity_dict["avalon_attrs"].get(self.id_cust_attr) + if not mongo_id: + continue + if mongo_id in all_mongo_ids: + same_mongo_id.append(mongo_id) + else: + all_mongo_ids[mongo_id] = [] + all_mongo_ids[mongo_id].append(ftrack_id) + + if avalon_project: + mongo_id = str(avalon_project["_id"]) + ftrack_avalon_mapper[self.ft_project_id] = mongo_id + avalon_ftrack_mapper[mongo_id] = self.ft_project_id + update_ftrack_ids.append(self.ft_project_id) + else: + create_ftrack_ids.append(self.ft_project_id) + + # make it go hierarchically + prepare_queue = queue.Queue() + + for child_id in self.entities_dict[self.ft_project_id]["children"]: + prepare_queue.put(child_id) + + while not prepare_queue.empty(): + ftrack_id = prepare_queue.get() + for child_id in self.entities_dict[ftrack_id]["children"]: + prepare_queue.put(child_id) + + entity_dict = self.entities_dict[ftrack_id] + ent_path = self.get_ent_path(ftrack_id) + + mongo_id = entity_dict["avalon_attrs"].get(self.id_cust_attr) + av_ent_by_mongo_id = self.avalon_ents_by_id.get(mongo_id) + if av_ent_by_mongo_id: + av_ent_ftrack_id = av_ent_by_mongo_id.get("data", {}).get( + "ftrackId" + ) + is_right = False + else_match_better = False + if av_ent_ftrack_id and av_ent_ftrack_id == ftrack_id: + is_right = True + + elif mongo_id not in same_mongo_id: + is_right = True + + else: + ftrack_ids_with_same_mongo = all_mongo_ids[mongo_id] + for _ftrack_id in ftrack_ids_with_same_mongo: + if _ftrack_id == av_ent_ftrack_id: + continue + + _entity_dict = self.entities_dict[_ftrack_id] + _mongo_id = _entity_dict["avalon_attrs"][ + self.id_cust_attr + ] + _av_ent_by_mongo_id = self.avalon_ents_by_id.get( + _mongo_id + ) + _av_ent_ftrack_id = _av_ent_by_mongo_id.get( + "data", {} + ).get("ftrackId") + if _av_ent_ftrack_id == ftrack_id: + else_match_better = True + break + + if not is_right and not else_match_better: + entity = entity_dict["entity"] + ent_path_items = [ent["name"] for ent in entity["link"]] + parents = ent_path_items[1:len(ent_path_items)-1:] + av_parents = av_ent_by_mongo_id["data"]["parents"] + if av_parents == parents: + is_right = True + else: + name = entity_dict["name"] + av_name = av_ent_by_mongo_id["name"] + if name == av_name: + is_right = True + + if is_right: + self.log.debug( + "Existing (by MongoID) <{}>".format(ent_path) + ) + ftrack_avalon_mapper[ftrack_id] = mongo_id + avalon_ftrack_mapper[mongo_id] = ftrack_id + update_ftrack_ids.append(ftrack_id) + continue + + mongo_id = self.avalon_ents_by_ftrack_id.get(ftrack_id) + if not mongo_id: + mongo_id = self.avalon_ents_by_name.get(entity_dict["name"]) + if mongo_id: + self.log.debug( + "Existing (by matching name) <{}>".format(ent_path) + ) + else: + self.log.debug( + "Existing (by FtrackID in mongo) <{}>".format(ent_path) + ) + + if mongo_id: + ftrack_avalon_mapper[ftrack_id] = mongo_id + avalon_ftrack_mapper[mongo_id] = ftrack_id + update_ftrack_ids.append(ftrack_id) + continue + + self.log.debug("New <{}>".format(ent_path)) + create_ftrack_ids.append(ftrack_id) + + deleted_entities = [] + for mongo_id in self.avalon_ents_by_id: + if mongo_id in avalon_ftrack_mapper: + continue + deleted_entities.append(mongo_id) + + av_ent = self.avalon_ents_by_id[mongo_id] + av_ent_path_items = [p for p in av_ent["data"]["parents"]] + av_ent_path_items.append(av_ent["name"]) + self.log.debug("Deleted <{}>".format("/".join(av_ent_path_items))) + + self.ftrack_avalon_mapper = ftrack_avalon_mapper + self.avalon_ftrack_mapper = avalon_ftrack_mapper + self.create_ftrack_ids = create_ftrack_ids + self.update_ftrack_ids = update_ftrack_ids + self.deleted_entities = deleted_entities + + self.log.debug(( + "Ftrack -> Avalon comparation: New <{}> " + "| Existing <{}> | Deleted <{}>" + ).format( + len(create_ftrack_ids), + len(update_ftrack_ids), + len(deleted_entities) + )) + + def filter_with_children(self, ftrack_id): + if ftrack_id not in self.entities_dict: + return + ent_dict = self.entities_dict[ftrack_id] + parent_id = ent_dict["parent_id"] + self.entities_dict[parent_id]["children"].remove(ftrack_id) + + children_queue = queue.Queue() + children_queue.put(ftrack_id) + while not children_queue.empty(): + _ftrack_id = children_queue.get() + entity_dict = self.entities_dict.pop(_ftrack_id, {"children": []}) + for child_id in entity_dict["children"]: + children_queue.put(child_id) + + def prepare_changes(self): + self.log.debug("* Preparing changes for avalon/ftrack") + hierarchy_changing_ids = [] + ignore_keys = collections.defaultdict(list) + + update_queue = queue.Queue() + for ftrack_id in self.update_ftrack_ids: + update_queue.put(ftrack_id) + + while not update_queue.empty(): + ftrack_id = update_queue.get() + if ftrack_id == self.ft_project_id: + changes = self.prepare_project_changes() + if changes: + self.updates[self.avalon_project_id] = changes + continue + + ftrack_ent_dict = self.entities_dict[ftrack_id] + + # *** check parents + parent_check = False + + ftrack_parent_id = ftrack_ent_dict["parent_id"] + avalon_id = self.ftrack_avalon_mapper[ftrack_id] + avalon_entity = self.avalon_ents_by_id[avalon_id] + avalon_parent_id = avalon_entity["data"]["visualParent"] + if avalon_parent_id is not None: + avalon_parent_id = str(avalon_parent_id) + + ftrack_parent_mongo_id = self.ftrack_avalon_mapper[ + ftrack_parent_id + ] + + # if parent is project + if (ftrack_parent_mongo_id == avalon_parent_id) or ( + ftrack_parent_id == self.ft_project_id and + avalon_parent_id is None + ): + parent_check = True + + # check name + ftrack_name = ftrack_ent_dict["name"] + avalon_name = avalon_entity["name"] + name_check = ftrack_name == avalon_name + + # IDEAL STATE: both parent and name check passed + if parent_check and name_check: + continue + + # If entity is changeable then change values of parent or name + if self.changeability_by_mongo_id[avalon_id]: + # TODO logging + if not parent_check: + if ftrack_parent_mongo_id == str(self.avalon_project_id): + new_parent_name = self.entities_dict[ + self.ft_project_id]["name"] + new_parent_id = None + else: + new_parent_name = self.avalon_ents_by_id[ + ftrack_parent_mongo_id]["name"] + new_parent_id = ObjectId(ftrack_parent_mongo_id) + + if avalon_parent_id == str(self.avalon_project_id): + old_parent_name = self.entities_dict[ + self.ft_project_id]["name"] + else: + old_parent_name = self.avalon_ents_by_id[ + ftrack_parent_mongo_id]["name"] + + self.updates[avalon_id]["data"] = { + "visualParent": new_parent_id + } + ignore_keys[ftrack_id].append("data.visualParent") + self.log.debug(( + "Avalon entity \"{}\" changed parent \"{}\" -> \"{}\"" + ).format(avalon_name, old_parent_name, new_parent_name)) + + if not name_check: + self.updates[avalon_id]["name"] = ftrack_name + ignore_keys[ftrack_id].append("name") + self.log.debug( + "Avalon entity \"{}\" was renamed to \"{}\"".format( + avalon_name, ftrack_name + ) + ) + continue + + # parents and hierarchy must be recalculated + hierarchy_changing_ids.append(ftrack_id) + + # Parent is project if avalon_parent_id is set to None + if avalon_parent_id is None: + avalon_parent_id = str(self.avalon_project_id) + + if not name_check: + ent_path = self.get_ent_path(ftrack_id) + # TODO report + # TODO logging + self.entities_dict[ftrack_id]["name"] = avalon_name + self.entities_dict[ftrack_id]["entity"]["name"] = ( + avalon_name + ) + self.entities_dict[ftrack_id]["final_entity"]["name"] = ( + avalon_name + ) + self.log.warning("Name was changed back to {} <{}>".format( + avalon_name, ent_path + )) + self._ent_pats_by_ftrack_id.pop(ftrack_id, None) + msg = ( + "NOTE: Allowed symbols are Letters( a-Z )," + " Numbers( 0-9 ) and Underscore( _ )
" + ) + }) + log_msgs = [] + for name, ids in self.failed_regex.items(): + error_title = { + "type": "label", + "value": "## {}".format(name) + } + items.append(error_title) + paths = [] + for entity_id in ids: + ent_path = self.get_ent_path(entity_id) + paths.append(ent_path) + + error_message = { + "type": "label", + "value": '{}
'.format("{}
'.format("NOTE: It is not allowed to have same name" + " for multiple entities in one project
" + ) + }) + log_msgs = [] + for name, ids in self.duplicates.items(): + error_title = { + "type": "label", + "value": "## {}".format(name) + } + items.append(error_title) + paths = [] + for entity_id in ids: + ent_path = self.get_ent_path(entity_id) + paths.append(ent_path) + + error_message = { + "type": "label", + "value": '{}
'.format("{}
'.format("{}
".format( + str(traceback.format_exc()).replace( + "\n", "