From a7ac81dcc10dc2b793f4acb5fa19480397621842 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 5 Dec 2018 16:58:39 +0100 Subject: [PATCH 1/7] Prepared action for event server --- pype/ftrack/events/action_sync_to_avalon.py | 366 ++++++++++++++++++++ 1 file changed, 366 insertions(+) create mode 100644 pype/ftrack/events/action_sync_to_avalon.py diff --git a/pype/ftrack/events/action_sync_to_avalon.py b/pype/ftrack/events/action_sync_to_avalon.py new file mode 100644 index 0000000000..e305b30739 --- /dev/null +++ b/pype/ftrack/events/action_sync_to_avalon.py @@ -0,0 +1,366 @@ +import sys +import argparse +import logging +import os +import ftrack_api +import json +import re +from pype import lib +from pype.ftrack.actions.ftrack_action_handler import BaseAction +from bson.objectid import ObjectId +from avalon import io, inventory + +from pype.ftrack import ftrack_utils + +class Sync_To_Avalon(BaseAction): + ''' + Synchronizing data action - from Ftrack to Avalon DB + + Stores all information about entity. + - Name(string) - Most important information = identifier of entity + - Parent(ObjectId) - Avalon Project Id, if entity is not project itself + - Silo(string) - Last parent except project + - Data(dictionary): + - VisualParent(ObjectId) - Avalon Id of parent asset + - Parents(array of string) - All parent names except project + - Tasks(array of string) - Tasks on asset + - FtrackId(string) + - entityType(string) - entity's type on Ftrack + * All Custom attributes in group 'Avalon' which name don't start with 'avalon_' + + * These information are stored also for all parents and children entities. + + Avalon ID of asset is stored to Ftrack -> Custom attribute 'avalon_mongo_id'. + - action IS NOT creating this Custom attribute if doesn't exist + - run 'Create Custom Attributes' action or do it manually (Not recommended) + + If Ftrack entity already has Custom Attribute 'avalon_mongo_id' that stores ID: + - name, parents and silo are checked -> shows error if are not exact the same + - after sync it is not allowed to change names or move entities + + If ID in 'avalon_mongo_id' is empty string or is not found in DB: + - tries to find entity by name + - found: + - raise error if ftrackId/visual parent/parents are not same + - not found: + - Creates asset/project + + ''' + + #: Action identifier. + identifier = 'sync.to.avalon' + #: Action label. + label = 'SyncToAvalon' + #: Action description. + description = 'Send data from Ftrack to Avalon' + #: Action icon. + icon = 'https://cdn1.iconfinder.com/data/icons/hawcons/32/699650-icon-92-inbox-download-512.png' + + def register(self): + '''Registers the action, subscribing the the discover and launch topics.''' + self.session.event_hub.subscribe( + 'topic=ftrack.action.discover', + self._discover + ) + + self.session.event_hub.subscribe( + 'topic=ftrack.action.launch and data.actionIdentifier={0}'.format( + self.identifier + ), + self._launch + ) + + self.log.info("Action '{}' - Registered successfully".format(self.__class__.__name__)) + + def discover(self, session, entities, event): + ''' Validation ''' + roleCheck = False + discover = False + roleList = ['Administrator', 'Project Manager'] + userId = event['source']['user']['id'] + user = session.query('User where id is ' + userId).one() + + for role in user['user_security_roles']: + if role['security_role']['name'] in roleList: + roleCheck = True + if roleCheck is True: + for entity in entities: + if entity.entity_type.lower() not in ['task', 'assetversion']: + discover = True + break + + return discover + + + def launch(self, session, entities, event): + message = "" + + # JOB SETTINGS + userId = event['source']['user']['id'] + user = session.query('User where id is ' + userId).one() + + job = session.create('Job', { + 'user': user, + 'status': 'running', + 'data': json.dumps({ + 'description': 'Synch Ftrack to Avalon.' + }) + }) + + try: + self.log.info("Action <" + self.__class__.__name__ + "> is running") + self.ca_mongoid = 'avalon_mongo_id' + #TODO AVALON_PROJECTS, AVALON_ASSET, AVALON_SILO should be set up otherwise console log shows avalon debug + self.setAvalonAttributes() + self.importable = [] + + # get from top entity in hierarchy all parent entities + top_entity = entities[0]['link'] + if len(top_entity) > 1: + for e in top_entity: + parent_entity = session.get(e['type'], e['id']) + self.importable.append(parent_entity) + + # get all child entities separately/unique + for entity in entities: + self.getShotAsset(entity) + + # Check names: REGEX in schema/duplicates - raise error if found + all_names = [] + duplicates = [] + + for e in self.importable: + ftrack_utils.avalon_check_name(e) + if e['name'] in all_names: + duplicates.append("'{}'".format(e['name'])) + else: + all_names.append(e['name']) + + if len(duplicates) > 0: + raise ValueError("Entity name duplication: {}".format(", ".join(duplicates))) + + ## ----- PROJECT ------ + # store Ftrack project- self.importable[0] must be project entity!!! + self.entityProj = self.importable[0] + # set AVALON_ env + os.environ["AVALON_PROJECT"] = self.entityProj["full_name"] + os.environ["AVALON_ASSET"] = self.entityProj["full_name"] + + self.avalon_project = None + + io.install() + + # Import all entities to Avalon DB + for e in self.importable: + self.importToAvalon(session, e) + + io.uninstall() + + job['status'] = 'done' + session.commit() + self.log.info('Synchronization to Avalon was successfull!') + + except ValueError as ve: + job['status'] = 'failed' + session.commit() + message = str(ve) + self.log.error('Error during syncToAvalon: {}'.format(message)) + + except Exception as e: + job['status'] = 'failed' + session.commit() + exc_type, exc_obj, exc_tb = sys.exc_info() + fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] + log_message = "{}/{}/Line: {}".format(exc_type, fname, exc_tb.tb_lineno) + self.log.error('Error during syncToAvalon: {}'.format(log_message)) + message = 'Unexpected Error - Please check Log for more information' + + if len(message) > 0: + message = "Unable to sync: {}".format(message) + return { + 'success': False, + 'message': message + } + + return { + 'success': True, + 'message': "Synchronization was successfull" + } + + def setAvalonAttributes(self): + self.custom_attributes = [] + all_avalon_attr = self.session.query('CustomAttributeGroup where name is "avalon"').one() + for cust_attr in all_avalon_attr['custom_attribute_configurations']: + if 'avalon_' not in cust_attr['key']: + self.custom_attributes.append(cust_attr) + + def getShotAsset(self, entity): + if not (entity.entity_type in ['Task']): + if entity not in self.importable: + self.importable.append(entity) + + if entity['children']: + childrens = entity['children'] + for child in childrens: + self.getShotAsset(child) + + def importToAvalon(self, session, entity): + # --- Begin: PUSH TO Avalon --- + + entity_type = entity.entity_type + + if entity_type.lower() in ['project']: + # Set project Config + config = ftrack_utils.get_config(entity) + # Set project template + template = lib.get_avalon_project_template_schema() + if self.ca_mongoid in entity['custom_attributes']: + try: + projectId = ObjectId(self.entityProj['custom_attributes'][self.ca_mongoid]) + self.avalon_project = io.find_one({"_id": projectId}) + except: + self.log.debug("Entity {} don't have stored entity id in ftrack".format(entity['name'])) + + if self.avalon_project is None: + self.avalon_project = io.find_one({ + "type": "project", + "name": entity["full_name"] + }) + if self.avalon_project is None: + inventory.save(entity['full_name'], config, template) + self.avalon_project = io.find_one({ + "type": "project", + "name": entity["full_name"] + }) + + elif self.avalon_project['name'] != entity['full_name']: + raise ValueError('You can\'t change name {} to {}, avalon DB won\'t work properly!'.format(self.avalon_project['name'], name)) + + data = ftrack_utils.get_data(self, entity, session,self.custom_attributes) + + # Store info about project (FtrackId) + io.update_many({ + 'type': 'project', + 'name': entity['full_name'] + }, { + '$set':{'data':data, 'config':config} + }) + + self.projectId = self.avalon_project["_id"] + if self.ca_mongoid in entity['custom_attributes']: + entity['custom_attributes'][self.ca_mongoid] = str(self.projectId) + else: + self.log.error('Custom attribute for "{}" is not created.'.format(entity['name'])) + return + + ## ----- ASSETS ------ + # Presets: + data = ftrack_utils.get_data(self, entity, session, self.custom_attributes) + + # return if entity is silo + if len(data['parents']) == 0: + return + else: + silo = data['parents'][0] + + os.environ['AVALON_SILO'] = silo + + name = entity['name'] + os.environ['AVALON_ASSET'] = name + + + # Try to find asset in current database + avalon_asset = None + if self.ca_mongoid in entity['custom_attributes']: + try: + entityId = ObjectId(entity['custom_attributes'][self.ca_mongoid]) + avalon_asset = io.find_one({"_id": entityId}) + except: + self.log.debug("Entity {} don't have stored entity id in ftrack".format(entity['name'])) + + if avalon_asset is None: + avalon_asset = io.find_one({'type': 'asset', 'name': name}) + # Create if don't exists + if avalon_asset is None: + inventory.create_asset(name, silo, data, self.projectId) + self.log.debug("Asset {} - created".format(name)) + + # Raise error if it seems to be different ent. with same name + elif (avalon_asset['data']['parents'] != data['parents'] or + avalon_asset['silo'] != silo): + raise ValueError('In Avalon DB already exists entity with name "{0}"'.format(name)) + + elif avalon_asset['name'] != entity['name']: + raise ValueError('You can\'t change name {} to {}, avalon DB won\'t work properly - please set name back'.format(avalon_asset['name'], name)) + elif avalon_asset['silo'] != silo or avalon_asset['data']['parents'] != data['parents']: + old_path = "/".join(avalon_asset['data']['parents']) + new_path = "/".join(data['parents']) + raise ValueError('You can\'t move with entities. Entity "{}" was moved from "{}" to "{}" '.format(avalon_asset['name'], old_path, new_path)) + + # Update info + io.update_many({'type': 'asset','name': name}, + {'$set':{'data':data, 'silo': silo}}) + + self.log.debug("Asset {} - updated".format(name)) + + entityId = io.find_one({'type': 'asset', 'name': name})['_id'] + ## FTRACK FEATURE - FTRACK MUST HAVE avalon_mongo_id FOR EACH ENTITY TYPE EXCEPT TASK + # Set custom attribute to avalon/mongo id of entity (parentID is last) + if self.ca_mongoid in entity['custom_attributes']: + entity['custom_attributes'][self.ca_mongoid] = str(entityId) + else: + self.log.error("Custom attribute for <{}> is not created.".format(entity['name'])) + + session.commit() + + +def register(session, **kw): + '''Register plugin. Called when used as an plugin.''' + + # Validate that session is an instance of ftrack_api.Session. If not, + # assume that register is being called from an old or incompatible API and + # return without doing anything. + if not isinstance(session, ftrack_api.session.Session): + return + + action_handler = Sync_To_Avalon(session) + action_handler.register() + + +def main(arguments=None): + '''Set up logging and register action.''' + if arguments is None: + arguments = [] + + parser = argparse.ArgumentParser() + # Allow setting of logging level from arguments. + loggingLevels = {} + for level in ( + logging.NOTSET, logging.DEBUG, logging.INFO, logging.WARNING, + logging.ERROR, logging.CRITICAL + ): + loggingLevels[logging.getLevelName(level).lower()] = level + + parser.add_argument( + '-v', '--verbosity', + help='Set the logging output verbosity.', + choices=loggingLevels.keys(), + default='info' + ) + namespace = parser.parse_args(arguments) + + # Set up basic logging + logging.basicConfig(level=loggingLevels[namespace.verbosity]) + + session = ftrack_api.Session() + register(session) + + # Wait for events + logging.info( + 'Registered actions and listening for events. Use Ctrl-C to abort.' + ) + session.event_hub.wait() + + +if __name__ == '__main__': + raise SystemExit(main(sys.argv[1:])) From 2da8a47442ad381c6b503b71c9919ab290599f1b Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 5 Dec 2018 19:00:29 +0100 Subject: [PATCH 2/7] preparation for 'ftrack_resources' --- pype/ftrack/actions/action_Apps.py | 10 +++++++++- pype/ftrack/actions/ftrack_action_handler.py | 17 +++++------------ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/pype/ftrack/actions/action_Apps.py b/pype/ftrack/actions/action_Apps.py index 3d1bf093de..084ffa9aec 100644 --- a/pype/ftrack/actions/action_Apps.py +++ b/pype/ftrack/actions/action_Apps.py @@ -31,11 +31,19 @@ def registerApp(app, session): label = apptoml['ftrack_label'] icon = None + ftrack_resources = "" # Path to resources here + if 'icon' in apptoml: icon = apptoml['icon'] + if '{ftrack_resources}' in icon: + icon = icon.format(ftrack_resources) + + description = None + if 'description' in apptoml: + description = apptoml['description'] # register action - AppAction(session, label, name, executable, variant, icon).register() + AppAction(session, label, name, executable, variant, icon, description).register() def register(session): diff --git a/pype/ftrack/actions/ftrack_action_handler.py b/pype/ftrack/actions/ftrack_action_handler.py index 15c57dbb1c..89fa669992 100644 --- a/pype/ftrack/actions/ftrack_action_handler.py +++ b/pype/ftrack/actions/ftrack_action_handler.py @@ -14,9 +14,6 @@ import acre from pype import api as pype -log = pype.Logger.getLogger(__name__, "ftrack") - -log.debug("pype.Anatomy: {}".format(pype.Anatomy)) class AppAction(object): @@ -231,13 +228,9 @@ class AppAction(object): entity, id = entities[0] entity = session.get(entity, id) - silo = "Film" - if entity.entity_type == "AssetBuild": - silo = "Asset" - # set environments for Avalon os.environ["AVALON_PROJECT"] = entity['project']['full_name'] - os.environ["AVALON_SILO"] = silo + os.environ["AVALON_SILO"] = entity['ancestors'][0]['name'] os.environ["AVALON_ASSET"] = entity['parent']['name'] os.environ["AVALON_TASK"] = entity['name'] os.environ["AVALON_APP"] = self.identifier @@ -262,7 +255,7 @@ class AppAction(object): try: anatomy = anatomy.format(data) except Exception as e: - log.error("{0} Error in anatomy.format: {1}".format(__name__, e)) + self.log.error("{0} Error in anatomy.format: {1}".format(__name__, e)) os.environ["AVALON_WORKDIR"] = os.path.join(anatomy.work.root, anatomy.work.folder) # TODO Add paths to avalon setup from tomls @@ -328,7 +321,7 @@ class AppAction(object): try: fp = open(execfile) except PermissionError as p: - log.error('Access denied on {0} - {1}'. + self.log.error('Access denied on {0} - {1}'. format(execfile, p)) return { 'success': False, @@ -338,7 +331,7 @@ class AppAction(object): fp.close() # check executable permission if not os.access(execfile, os.X_OK): - log.error('No executable permission on {}'. + self.log.error('No executable permission on {}'. format(execfile)) return { 'success': False, @@ -347,7 +340,7 @@ class AppAction(object): } pass else: - log.error('Launcher doesn\'t exist - {}'. + self.log.error('Launcher doesn\'t exist - {}'. format(execfile)) return { 'success': False, From 3fda5c32b3d9761ab3981d007b55bf53409e59cb Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 6 Dec 2018 10:13:47 +0100 Subject: [PATCH 3/7] Action syncToAvalon - changed identifier and label. Is visible only for role 'Pypeclub' --- pype/ftrack/actions/action_syncToAvalon.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pype/ftrack/actions/action_syncToAvalon.py b/pype/ftrack/actions/action_syncToAvalon.py index cad43684c9..04e9ed53a5 100644 --- a/pype/ftrack/actions/action_syncToAvalon.py +++ b/pype/ftrack/actions/action_syncToAvalon.py @@ -48,9 +48,9 @@ class SyncToAvalon(BaseAction): ''' #: Action identifier. - identifier = 'sync.to.avalon' + identifier = 'sync.to.avalon.local' #: Action label. - label = 'SyncToAvalon' + label = 'SyncToAvalon - Local' #: Action description. description = 'Send data from Ftrack to Avalon' #: Action icon. @@ -61,7 +61,7 @@ class SyncToAvalon(BaseAction): ''' Validation ''' roleCheck = False discover = False - roleList = ['Administrator', 'Project Manager'] + roleList = ['Pypeclub'] userId = event['source']['user']['id'] user = session.query('User where id is ' + userId).one() From f59bcf2e44eec0708649b4fdd9ce45e517829055 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 6 Dec 2018 12:57:44 +0100 Subject: [PATCH 4/7] renamed filename --- .../{action_syncToAvalon.py => action_sync_to_avalon_local.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pype/ftrack/actions/{action_syncToAvalon.py => action_sync_to_avalon_local.py} (100%) diff --git a/pype/ftrack/actions/action_syncToAvalon.py b/pype/ftrack/actions/action_sync_to_avalon_local.py similarity index 100% rename from pype/ftrack/actions/action_syncToAvalon.py rename to pype/ftrack/actions/action_sync_to_avalon_local.py From e19fbea3e9fb18c243a05ce034308e33eceb75ad Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 6 Dec 2018 15:20:05 +0100 Subject: [PATCH 5/7] Identifier is now full app name(with version), so same app won't launch twice if two versions are available. --- pype/ftrack/actions/action_Apps.py | 3 ++- pype/ftrack/actions/ftrack_action_handler.py | 4 +--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pype/ftrack/actions/action_Apps.py b/pype/ftrack/actions/action_Apps.py index 3d1bf093de..76c6ba1e06 100644 --- a/pype/ftrack/actions/action_Apps.py +++ b/pype/ftrack/actions/action_Apps.py @@ -9,7 +9,7 @@ from app.api import Logger log = Logger.getLogger(__name__) def registerApp(app, session): - name = app['name'].split("_")[0] + name = app['name'].replace("_", ".") variant = "" try: variant = app['name'].split("_")[1] @@ -59,6 +59,7 @@ def register(session): appNames.append(app['name']) apps.append(app) + apps = sorted(apps, key=lambda x: x['name']) for app in apps: try: registerApp(app, session) diff --git a/pype/ftrack/actions/ftrack_action_handler.py b/pype/ftrack/actions/ftrack_action_handler.py index 15c57dbb1c..63561951d4 100644 --- a/pype/ftrack/actions/ftrack_action_handler.py +++ b/pype/ftrack/actions/ftrack_action_handler.py @@ -72,9 +72,7 @@ class AppAction(object): ), self._launch ) - self.log.info("Application '{}' - Registered successfully".format(self.label)) - - self.log.info("Application '{}' - Registered successfully".format(self.label)) + self.log.info("Application '{} {}' - Registered successfully".format(self.label,self.variant)) def _discover(self, event): args = self._translate_event( From 3b3a379b7a722a8438af091cbaf63e9832cd3a09 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Fri, 7 Dec 2018 01:12:34 +0100 Subject: [PATCH 6/7] fixing issue with apps not launchign at all after last merge --- pype/ftrack/actions/action_Apps.py | 3 ++- pype/ftrack/actions/ftrack_action_handler.py | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pype/ftrack/actions/action_Apps.py b/pype/ftrack/actions/action_Apps.py index 76c6ba1e06..23667290bd 100644 --- a/pype/ftrack/actions/action_Apps.py +++ b/pype/ftrack/actions/action_Apps.py @@ -9,7 +9,7 @@ from app.api import Logger log = Logger.getLogger(__name__) def registerApp(app, session): - name = app['name'].replace("_", ".") + name = app['name'] variant = "" try: variant = app['name'].split("_")[1] @@ -17,6 +17,7 @@ def registerApp(app, session): log.warning("'{0}' - App 'name' and 'variant' is not separated by '_' (variant is not set)".format(app['name'])) return + log.warning("app name {}".format(name)) abspath = lib.which_app(app['name']) if abspath == None: log.error("'{0}' - App don't have config toml file".format(app['name'])) diff --git a/pype/ftrack/actions/ftrack_action_handler.py b/pype/ftrack/actions/ftrack_action_handler.py index 63561951d4..fb49c40464 100644 --- a/pype/ftrack/actions/ftrack_action_handler.py +++ b/pype/ftrack/actions/ftrack_action_handler.py @@ -135,7 +135,7 @@ class AppAction(object): else: apps = [] for app in project['config']['apps']: - apps.append(app['name'].split("_")[0]) + apps.append(app['name']) if self.identifier not in apps: return False @@ -238,8 +238,8 @@ class AppAction(object): os.environ["AVALON_SILO"] = silo os.environ["AVALON_ASSET"] = entity['parent']['name'] os.environ["AVALON_TASK"] = entity['name'] - os.environ["AVALON_APP"] = self.identifier - os.environ["AVALON_APP_NAME"] = self.identifier + "_" + self.variant + os.environ["AVALON_APP"] = self.identifier.split("_")[0] + os.environ["AVALON_APP_NAME"] = self.identifier os.environ["FTRACK_TASKID"] = id @@ -280,7 +280,7 @@ class AppAction(object): parents.append(session.get(item['type'], item['id'])) # collect all the 'environment' attributes from parents - tools_attr = [os.environ["AVALON_APP_NAME"]] + tools_attr = [os.environ["AVALON_APP"], os.environ["AVALON_APP_NAME"]] for parent in reversed(parents): # check if the attribute is empty, if not use it if parent['custom_attributes']['tools_env']: From 3a38a907847f126df819da80f7e9b00ad3e23a8d Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Fri, 7 Dec 2018 09:24:07 +0100 Subject: [PATCH 7/7] ftrackRun renamed to ftrack_run --- pype/ftrack/{ftrackRun.py => ftrack_run.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pype/ftrack/{ftrackRun.py => ftrack_run.py} (100%) diff --git a/pype/ftrack/ftrackRun.py b/pype/ftrack/ftrack_run.py similarity index 100% rename from pype/ftrack/ftrackRun.py rename to pype/ftrack/ftrack_run.py