From 4dd95fba6842d2b4c4556e2465cb2ef00f70cb1f Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 17 Mar 2022 19:22:35 +0100 Subject: [PATCH] added job and report messages --- .../action_fill_workfile_attr.py | 319 ++++++++++++++---- 1 file changed, 262 insertions(+), 57 deletions(-) diff --git a/openpype/modules/ftrack/event_handlers_user/action_fill_workfile_attr.py b/openpype/modules/ftrack/event_handlers_user/action_fill_workfile_attr.py index a72b29bdbe..77f18c49c1 100644 --- a/openpype/modules/ftrack/event_handlers_user/action_fill_workfile_attr.py +++ b/openpype/modules/ftrack/event_handlers_user/action_fill_workfile_attr.py @@ -1,4 +1,9 @@ +import os +import sys +import json import collections +import tempfile +import datetime import ftrack_api @@ -13,6 +18,8 @@ from openpype.lib import ( from openpype_modules.ftrack.lib import BaseAction, statics_icon from openpype_modules.ftrack.lib.avalon_sync import create_chunks +NOT_SYNCHRONIZED_TITLE = "Not synchronized" + class FillWorkfileAttributeAction(BaseAction): """Action fill work filename into custom attribute on tasks. @@ -44,24 +51,24 @@ class FillWorkfileAttributeAction(BaseAction): return is_valid def launch(self, session, entities, event): - task_entities = [] - other_entities = [] + # Separate entities and get project entity project_entity = None - project_selected = False for entity in entities: if project_entity is None: project_entity = self.get_project_from_entity(entity) - - ent_type_low = entity.entity_type.lower() - if ent_type_low == "project": - project_selected = True break - elif ent_type_low == "task": - task_entities.append(entity) - else: - other_entities.append(entity) + if not project_entity: + return { + "message": ( + "Couldn't find project entity." + " Could be an issue with permissions." + ), + "success": False + } + # Get project settings and check if custom attribute where workfile + # should be set is defined. project_name = project_entity["full_name"] project_settings = get_project_settings(project_name) custom_attribute_key = ( @@ -77,12 +84,16 @@ class FillWorkfileAttributeAction(BaseAction): "message": "Custom attribute key is not set in settings" } + # Try to find the custom attribute + # - get Task type object id task_obj_type = session.query( "select id from ObjectType where name is \"Task\"" ).one() + # - get text custom attribute type text_type = session.query( "select id from CustomAttributeType where name is \"text\"" ).one() + # - find the attribute attr_conf = session.query( ( "select id, key from CustomAttributeConfiguration" @@ -101,33 +112,184 @@ class FillWorkfileAttributeAction(BaseAction): ).format(custom_attribute_key) } + # Store report information + report = collections.defaultdict(list) + user_entity = session.query( + "User where id is {}".format(event["source"]["user"]["id"]) + ).one() + job_entity = session.create("Job", { + "user": user_entity, + "status": "running", + "data": json.dumps({ + "description": "(0/3) Fill of workfiles started" + }) + }) + session.commit() + + try: + self.in_job_process( + session, + entities, + job_entity, + project_entity, + project_settings, + attr_conf, + report + ) + except Exception: + self.log.error( + "Fill of workfiles to custom attribute failed", exc_info=True + ) + session.rollback() + + description = "Fill of workfiles Failed (Download traceback)" + self.add_traceback_to_job( + job_entity, session, sys.exc_info(), description + ) + return { + "message": ( + "Fill of workfiles failed." + " Check job for more information" + ), + "success": False + } + + job_entity["status"] = "done" + job_entity["data"] = json.dumps({ + "description": "Fill of workfiles completed." + }) + session.commit() + if report: + temp_obj = tempfile.NamedTemporaryFile( + mode="w", + prefix="openpype_ftrack_", + suffix=".json", + delete=False + ) + temp_obj.close() + temp_filepath = temp_obj.name + with open(temp_filepath, "w") as temp_file: + json.dump(report, temp_file) + + component_name = "{}_{}".format( + "FillWorkfilesReport", + datetime.datetime.now().strftime("%y-%m-%d-%H%M") + ) + self.add_file_component_to_job( + job_entity, session, temp_filepath, component_name + ) + # Delete temp file + os.remove(temp_filepath) + self._show_report(event, report, project_name) + return { + "message": ( + "Fill of workfiles finished with few issues." + " Check job for more information" + ), + "success": True + } + + return { + "success": True, + "message": "Finished with filling of work filenames" + } + + def _show_report(self, event, report, project_name): + items = [] + title = "Fill workfiles report ({}):".format(project_name) + + for subtitle, lines in report.items(): + if items: + items.append({ + "type": "label", + "value": "---" + }) + items.append({ + "type": "label", + "value": "# {}".format(subtitle) + }) + items.append({ + "type": "label", + "value": '

{}

'.format("
".join(lines)) + }) + + self.show_interface( + items=items, + title=title, + event=event + ) + + def in_job_process( + self, + session, + entities, + job_entity, + project_entity, + project_settings, + attr_conf, + report + ): + task_entities = [] + other_entities = [] + project_selected = False + for entity in entities: + ent_type_low = entity.entity_type.lower() + if ent_type_low == "project": + project_selected = True + break + + elif ent_type_low == "task": + task_entities.append(entity) + else: + other_entities.append(entity) + + project_name = project_entity["full_name"] + + # Find matchin asset documents and map them by ftrack task entities + # - result stored to 'asset_docs_with_task_entities' is list with + # tuple `(asset document, [task entitis, ...])` dbcon = AvalonMongoDB() dbcon.Session["AVALON_PROJECT"] = project_name + # Quety all asset documents asset_docs = list(dbcon.find({"type": "asset"})) + job_entity["data"] = json.dumps({ + "description": "(1/3) Asset documents queried." + }) + session.commit() + + # When project is selected then we can query whole project if project_selected: - asset_docs_with_task_names = self._get_asset_docs_for_project( - session, project_entity, asset_docs + asset_docs_with_task_entities = self._get_asset_docs_for_project( + session, project_entity, asset_docs, report ) else: - asset_docs_with_task_names = self._get_tasks_for_selection( - session, other_entities, task_entities, asset_docs + asset_docs_with_task_entities = self._get_tasks_for_selection( + session, other_entities, task_entities, asset_docs, report ) + job_entity["data"] = json.dumps({ + "description": "(2/3) Queried related task entities." + }) + session.commit() + + # Keep placeholders in the template unfilled host_name = "{host}" + extension = "{ext}" project_doc = dbcon.find_one({"type": "project"}) project_settings = get_project_settings(project_name) anatomy = Anatomy(project_name) templates_by_key = {} operations = [] - for asset_doc, task_entities in asset_docs_with_task_names: + for asset_doc, task_entities in asset_docs_with_task_entities: for task_entity in task_entities: workfile_data = get_workdir_data( project_doc, asset_doc, task_entity["name"], host_name ) + # Use version 1 for each workfile workfile_data["version"] = 1 - workfile_data["ext"] = "{ext}" + workfile_data["ext"] = extension task_type = workfile_data["task"]["type"] template_key = get_workfile_template_key( @@ -166,22 +328,40 @@ class FillWorkfileAttributeAction(BaseAction): session.recorded_operations.push(op) session.commit() - return True + job_entity["data"] = json.dumps({ + "description": "(3/3) Set custom attribute values." + }) + session.commit() + + def _get_entity_path(self, entity): + path_items = [] + for item in entity["link"]: + if item["type"].lower() != "project": + path_items.append(item["name"]) + return "/".join(path_items) + + def _get_asset_docs_for_project( + self, session, project_entity, asset_docs, report + ): + asset_docs_task_names = {} - def _get_asset_docs_for_project(self, session, project_entity, asset_docs): - asset_docs_task_names = collections.defaultdict(list) for asset_doc in asset_docs: asset_data = asset_doc["data"] - asset_tasks = asset_data.get("tasks") ftrack_id = asset_data.get("ftrackId") - if not asset_tasks or not ftrack_id: + if not ftrack_id: + hierarchy = list(asset_data.get("parents") or []) + hierarchy.append(asset_doc["name"]) + path = "/".join(hierarchy) + report[NOT_SYNCHRONIZED_TITLE].append(path) continue - asset_docs_task_names[ftrack_id].append( - (asset_doc, list(asset_tasks.keys())) + + asset_tasks = asset_data.get("tasks") or {} + asset_docs_task_names[ftrack_id] = ( + asset_doc, list(asset_tasks.keys()) ) task_entities = session.query(( - "select id, name, parent_id from Task where project_id is {}" + "select id, name, parent_id, link from Task where project_id is {}" ).format(project_entity["id"])).all() task_entities_by_parent_id = collections.defaultdict(list) for task_entity in task_entities: @@ -189,21 +369,23 @@ class FillWorkfileAttributeAction(BaseAction): task_entities_by_parent_id[parent_id].append(task_entity) output = [] - for ftrack_id, items in asset_docs_task_names.items(): - for item in items: - asset_doc, task_names = item - valid_task_entities = [] - for task_entity in task_entities_by_parent_id[ftrack_id]: - if task_entity["name"] in task_names: - valid_task_entities.append(task_entity) + for ftrack_id, item in asset_docs_task_names.items(): + asset_doc, task_names = item + valid_task_entities = [] + for task_entity in task_entities_by_parent_id[ftrack_id]: + if task_entity["name"] in task_names: + valid_task_entities.append(task_entity) + else: + path = self._get_entity_path(task_entity) + report[NOT_SYNCHRONIZED_TITLE].append(path) - if valid_task_entities: - output.append((asset_doc, valid_task_entities)) + if valid_task_entities: + output.append((asset_doc, valid_task_entities)) return output def _get_tasks_for_selection( - self, session, other_entities, task_entities, asset_docs + self, session, other_entities, task_entities, asset_docs, report ): all_tasks = object() asset_docs_by_ftrack_id = {} @@ -216,13 +398,13 @@ class FillWorkfileAttributeAction(BaseAction): if ftrack_id: asset_docs_by_ftrack_id[ftrack_id] = asset_doc - missing_docs = set() + missing_doc_ftrack_ids = {} all_tasks_ids = set() task_names_by_ftrack_id = collections.defaultdict(list) for other_entity in other_entities: ftrack_id = other_entity["id"] if ftrack_id not in asset_docs_by_ftrack_id: - missing_docs.add(ftrack_id) + missing_doc_ftrack_ids[ftrack_id] = None continue all_tasks_ids.add(ftrack_id) task_names_by_ftrack_id[ftrack_id] = all_tasks @@ -230,21 +412,18 @@ class FillWorkfileAttributeAction(BaseAction): for task_entity in task_entities: parent_id = task_entity["parent_id"] if parent_id not in asset_docs_by_ftrack_id: - missing_docs.add(parent_id) + missing_doc_ftrack_ids[parent_id] = None continue if all_tasks_ids not in all_tasks_ids: task_names_by_ftrack_id[ftrack_id].append(task_entity["name"]) ftrack_ids = set() - asset_doc_with_task_names_by_id = collections.defaultdict(list) + asset_doc_with_task_names_by_id = {} for ftrack_id, task_names in task_names_by_ftrack_id.items(): asset_doc = asset_docs_by_ftrack_id[ftrack_id] asset_data = asset_doc["data"] - asset_tasks = asset_data.get("tasks") - if not asset_tasks: - # TODO add to report - continue + asset_tasks = asset_data.get("tasks") or {} if task_names is all_tasks: task_names = list(asset_tasks.keys()) @@ -253,15 +432,19 @@ class FillWorkfileAttributeAction(BaseAction): for task_name in task_names: if task_name in asset_tasks: new_task_names.append(task_name) - else: - # TODO add report - pass + continue + + if ftrack_id not in missing_doc_ftrack_ids: + missing_doc_ftrack_ids[ftrack_id] = [] + if missing_doc_ftrack_ids[ftrack_id] is not None: + missing_doc_ftrack_ids[ftrack_id].append(task_name) + task_names = new_task_names if task_names: ftrack_ids.add(ftrack_id) - asset_doc_with_task_names_by_id[ftrack_id].append( - (asset_doc, task_names) + asset_doc_with_task_names_by_id[ftrack_id] = ( + asset_doc, task_names ) task_entities = session.query(( @@ -273,15 +456,37 @@ class FillWorkfileAttributeAction(BaseAction): task_entitiy_by_parent_id[parent_id].append(task_entity) output = [] - for ftrack_id, items in asset_doc_with_task_names_by_id.items(): - for item in items: - asset_doc, task_names = item - valid_task_entities = [] - for task_entity in task_entitiy_by_parent_id[ftrack_id]: - if task_entity["name"] in task_names: - valid_task_entities.append(task_entity) - if valid_task_entities: - output.append((asset_doc, valid_task_entities)) + for ftrack_id, item in asset_doc_with_task_names_by_id.items(): + asset_doc, task_names = item + valid_task_entities = [] + for task_entity in task_entitiy_by_parent_id[ftrack_id]: + if task_entity["name"] in task_names: + valid_task_entities.append(task_entity) + else: + if ftrack_id not in missing_doc_ftrack_ids: + missing_doc_ftrack_ids[ftrack_id] = [] + if missing_doc_ftrack_ids[ftrack_id] is not None: + missing_doc_ftrack_ids[ftrack_id].append(task_name) + if valid_task_entities: + output.append((asset_doc, valid_task_entities)) + + # Store report information about not synchronized entities + if missing_doc_ftrack_ids: + missing_entities = session.query( + "select id, link from TypedContext where id in ({})".format( + self.join_query_keys(missing_doc_ftrack_ids.keys()) + ) + ).all() + for missing_entity in missing_entities: + path = self._get_entity_path(missing_entity) + task_names = missing_doc_ftrack_ids[missing_entity["id"]] + if task_names is None: + report[NOT_SYNCHRONIZED_TITLE].append(path) + else: + for task_name in task_names: + task_path = "/".join([path, task_name]) + report[NOT_SYNCHRONIZED_TITLE].append(task_path) + return output