mirror of
https://github.com/ynput/ayon-core.git
synced 2025-12-25 21:32:15 +01:00
Merged in feature/PYPE-506_sync_to_avalon_action (pull request #373)
feature/PYPE-506_sync_to_avalon_action Approved-by: Milan Kolar <milan@orbi.tools>
This commit is contained in:
Jakub Trllo
Milan Kolar
commit
3cf76abf9d
6 changed files with 4656 additions and 1302 deletions
|
|
@ -1,351 +0,0 @@
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
import json
|
|
||||||
import argparse
|
|
||||||
import logging
|
|
||||||
import collections
|
|
||||||
|
|
||||||
from pype.vendor import ftrack_api
|
|
||||||
from pype.ftrack import BaseAction, lib
|
|
||||||
from pype.ftrack.lib.io_nonsingleton import DbConnector
|
|
||||||
from bson.objectid import ObjectId
|
|
||||||
|
|
||||||
|
|
||||||
class SyncHierarchicalAttrs(BaseAction):
|
|
||||||
|
|
||||||
db_con = DbConnector()
|
|
||||||
ca_mongoid = lib.get_ca_mongoid()
|
|
||||||
|
|
||||||
#: Action identifier.
|
|
||||||
identifier = 'sync.hierarchical.attrs.local'
|
|
||||||
#: Action label.
|
|
||||||
label = "Pype Admin"
|
|
||||||
variant = '- Sync Hier Attrs (Local)'
|
|
||||||
#: Action description.
|
|
||||||
description = 'Synchronize hierarchical attributes'
|
|
||||||
#: Icon
|
|
||||||
icon = '{}/ftrack/action_icons/PypeAdmin.svg'.format(
|
|
||||||
os.environ.get('PYPE_STATICS_SERVER', '')
|
|
||||||
)
|
|
||||||
|
|
||||||
#: roles that are allowed to register this action
|
|
||||||
role_list = ['Pypeclub', 'Administrator', 'Project Manager']
|
|
||||||
|
|
||||||
def discover(self, session, entities, event):
|
|
||||||
''' Validation '''
|
|
||||||
for entity in entities:
|
|
||||||
if (
|
|
||||||
entity.get('context_type', '').lower() in ('show', 'task') and
|
|
||||||
entity.entity_type.lower() != 'task'
|
|
||||||
):
|
|
||||||
return True
|
|
||||||
return False
|
|
||||||
|
|
||||||
def launch(self, session, entities, event):
|
|
||||||
self.interface_messages = {}
|
|
||||||
user = session.query(
|
|
||||||
'User where id is "{}"'.format(event['source']['user']['id'])
|
|
||||||
).one()
|
|
||||||
|
|
||||||
job = session.create('Job', {
|
|
||||||
'user': user,
|
|
||||||
'status': 'running',
|
|
||||||
'data': json.dumps({
|
|
||||||
'description': 'Sync Hierachical attributes'
|
|
||||||
})
|
|
||||||
})
|
|
||||||
session.commit()
|
|
||||||
self.log.debug('Job with id "{}" created'.format(job['id']))
|
|
||||||
|
|
||||||
process_session = ftrack_api.Session(
|
|
||||||
server_url=session.server_url,
|
|
||||||
api_key=session.api_key,
|
|
||||||
api_user=session.api_user,
|
|
||||||
auto_connect_event_hub=True
|
|
||||||
)
|
|
||||||
|
|
||||||
try:
|
|
||||||
# Collect hierarchical attrs
|
|
||||||
self.log.debug('Collecting Hierarchical custom attributes started')
|
|
||||||
custom_attributes = {}
|
|
||||||
all_avalon_attr = process_session.query(
|
|
||||||
'CustomAttributeGroup where name is "avalon"'
|
|
||||||
).one()
|
|
||||||
|
|
||||||
error_key = (
|
|
||||||
'Hierarchical attributes with set "default" value (not allowed)'
|
|
||||||
)
|
|
||||||
|
|
||||||
for cust_attr in all_avalon_attr['custom_attribute_configurations']:
|
|
||||||
if 'avalon_' in cust_attr['key']:
|
|
||||||
continue
|
|
||||||
|
|
||||||
if not cust_attr['is_hierarchical']:
|
|
||||||
continue
|
|
||||||
|
|
||||||
if cust_attr['default']:
|
|
||||||
if error_key not in self.interface_messages:
|
|
||||||
self.interface_messages[error_key] = []
|
|
||||||
self.interface_messages[error_key].append(
|
|
||||||
cust_attr['label']
|
|
||||||
)
|
|
||||||
|
|
||||||
self.log.warning((
|
|
||||||
'Custom attribute "{}" has set default value.'
|
|
||||||
' This attribute can\'t be synchronized'
|
|
||||||
).format(cust_attr['label']))
|
|
||||||
continue
|
|
||||||
|
|
||||||
custom_attributes[cust_attr['key']] = cust_attr
|
|
||||||
|
|
||||||
self.log.debug(
|
|
||||||
'Collecting Hierarchical custom attributes has finished'
|
|
||||||
)
|
|
||||||
|
|
||||||
if not custom_attributes:
|
|
||||||
msg = 'No hierarchical attributes to sync.'
|
|
||||||
self.log.debug(msg)
|
|
||||||
return {
|
|
||||||
'success': True,
|
|
||||||
'message': msg
|
|
||||||
}
|
|
||||||
|
|
||||||
entity = entities[0]
|
|
||||||
if entity.entity_type.lower() == 'project':
|
|
||||||
project_name = entity['full_name']
|
|
||||||
else:
|
|
||||||
project_name = entity['project']['full_name']
|
|
||||||
|
|
||||||
self.db_con.install()
|
|
||||||
self.db_con.Session['AVALON_PROJECT'] = project_name
|
|
||||||
|
|
||||||
_entities = self._get_entities(event, process_session)
|
|
||||||
|
|
||||||
for entity in _entities:
|
|
||||||
self.log.debug(30*'-')
|
|
||||||
self.log.debug(
|
|
||||||
'Processing entity "{}"'.format(entity.get('name', entity))
|
|
||||||
)
|
|
||||||
|
|
||||||
ent_name = entity.get('name', entity)
|
|
||||||
if entity.entity_type.lower() == 'project':
|
|
||||||
ent_name = entity['full_name']
|
|
||||||
|
|
||||||
for key in custom_attributes:
|
|
||||||
self.log.debug(30*'*')
|
|
||||||
self.log.debug(
|
|
||||||
'Processing Custom attribute key "{}"'.format(key)
|
|
||||||
)
|
|
||||||
# check if entity has that attribute
|
|
||||||
if key not in entity['custom_attributes']:
|
|
||||||
error_key = 'Missing key on entities'
|
|
||||||
if error_key not in self.interface_messages:
|
|
||||||
self.interface_messages[error_key] = []
|
|
||||||
|
|
||||||
self.interface_messages[error_key].append(
|
|
||||||
'- key: "{}" - entity: "{}"'.format(key, ent_name)
|
|
||||||
)
|
|
||||||
|
|
||||||
self.log.error((
|
|
||||||
'- key "{}" not found on "{}"'
|
|
||||||
).format(key, ent_name))
|
|
||||||
continue
|
|
||||||
|
|
||||||
value = self.get_hierarchical_value(key, entity)
|
|
||||||
if value is None:
|
|
||||||
error_key = (
|
|
||||||
'Missing value for key on entity'
|
|
||||||
' and its parents (synchronization was skipped)'
|
|
||||||
)
|
|
||||||
if error_key not in self.interface_messages:
|
|
||||||
self.interface_messages[error_key] = []
|
|
||||||
|
|
||||||
self.interface_messages[error_key].append(
|
|
||||||
'- key: "{}" - entity: "{}"'.format(key, ent_name)
|
|
||||||
)
|
|
||||||
|
|
||||||
self.log.warning((
|
|
||||||
'- key "{}" not set on "{}" or its parents'
|
|
||||||
).format(key, ent_name))
|
|
||||||
continue
|
|
||||||
|
|
||||||
self.update_hierarchical_attribute(entity, key, value)
|
|
||||||
|
|
||||||
job['status'] = 'done'
|
|
||||||
session.commit()
|
|
||||||
|
|
||||||
except Exception:
|
|
||||||
self.log.error(
|
|
||||||
'Action "{}" failed'.format(self.label),
|
|
||||||
exc_info=True
|
|
||||||
)
|
|
||||||
|
|
||||||
finally:
|
|
||||||
self.db_con.uninstall()
|
|
||||||
|
|
||||||
if job['status'] in ('queued', 'running'):
|
|
||||||
job['status'] = 'failed'
|
|
||||||
session.commit()
|
|
||||||
if self.interface_messages:
|
|
||||||
title = "Errors during SyncHierarchicalAttrs"
|
|
||||||
self.show_interface_from_dict(
|
|
||||||
messages=self.interface_messages, title=title, event=event
|
|
||||||
)
|
|
||||||
|
|
||||||
return True
|
|
||||||
|
|
||||||
def get_hierarchical_value(self, key, entity):
|
|
||||||
value = entity['custom_attributes'][key]
|
|
||||||
if (
|
|
||||||
value is not None or
|
|
||||||
entity.entity_type.lower() == 'project'
|
|
||||||
):
|
|
||||||
return value
|
|
||||||
|
|
||||||
return self.get_hierarchical_value(key, entity['parent'])
|
|
||||||
|
|
||||||
def update_hierarchical_attribute(self, entity, key, value):
|
|
||||||
if (
|
|
||||||
entity['context_type'].lower() not in ('show', 'task') or
|
|
||||||
entity.entity_type.lower() == 'task'
|
|
||||||
):
|
|
||||||
return
|
|
||||||
|
|
||||||
ent_name = entity.get('name', entity)
|
|
||||||
if entity.entity_type.lower() == 'project':
|
|
||||||
ent_name = entity['full_name']
|
|
||||||
|
|
||||||
hierarchy = '/'.join(
|
|
||||||
[a['name'] for a in entity.get('ancestors', [])]
|
|
||||||
)
|
|
||||||
if hierarchy:
|
|
||||||
hierarchy = '/'.join(
|
|
||||||
[entity['project']['full_name'], hierarchy, entity['name']]
|
|
||||||
)
|
|
||||||
elif entity.entity_type.lower() == 'project':
|
|
||||||
hierarchy = entity['full_name']
|
|
||||||
else:
|
|
||||||
hierarchy = '/'.join(
|
|
||||||
[entity['project']['full_name'], entity['name']]
|
|
||||||
)
|
|
||||||
|
|
||||||
self.log.debug('- updating entity "{}"'.format(hierarchy))
|
|
||||||
|
|
||||||
# collect entity's custom attributes
|
|
||||||
custom_attributes = entity.get('custom_attributes')
|
|
||||||
if not custom_attributes:
|
|
||||||
return
|
|
||||||
|
|
||||||
mongoid = custom_attributes.get(self.ca_mongoid)
|
|
||||||
if not mongoid:
|
|
||||||
error_key = 'Missing MongoID on entities (try SyncToAvalon first)'
|
|
||||||
if error_key not in self.interface_messages:
|
|
||||||
self.interface_messages[error_key] = []
|
|
||||||
|
|
||||||
if ent_name not in self.interface_messages[error_key]:
|
|
||||||
self.interface_messages[error_key].append(ent_name)
|
|
||||||
|
|
||||||
self.log.warning(
|
|
||||||
'-- entity "{}" is not synchronized to avalon. Skipping'.format(
|
|
||||||
ent_name
|
|
||||||
)
|
|
||||||
)
|
|
||||||
return
|
|
||||||
|
|
||||||
try:
|
|
||||||
mongoid = ObjectId(mongoid)
|
|
||||||
except Exception:
|
|
||||||
error_key = 'Invalid MongoID on entities (try SyncToAvalon)'
|
|
||||||
if error_key not in self.interface_messages:
|
|
||||||
self.interface_messages[error_key] = []
|
|
||||||
|
|
||||||
if ent_name not in self.interface_messages[error_key]:
|
|
||||||
self.interface_messages[error_key].append(ent_name)
|
|
||||||
|
|
||||||
self.log.warning(
|
|
||||||
'-- entity "{}" has stored invalid MongoID. Skipping'.format(
|
|
||||||
ent_name
|
|
||||||
)
|
|
||||||
)
|
|
||||||
return
|
|
||||||
# Find entity in Mongo DB
|
|
||||||
mongo_entity = self.db_con.find_one({'_id': mongoid})
|
|
||||||
if not mongo_entity:
|
|
||||||
error_key = 'Entities not found in Avalon DB (try SyncToAvalon)'
|
|
||||||
if error_key not in self.interface_messages:
|
|
||||||
self.interface_messages[error_key] = []
|
|
||||||
|
|
||||||
if ent_name not in self.interface_messages[error_key]:
|
|
||||||
self.interface_messages[error_key].append(ent_name)
|
|
||||||
|
|
||||||
self.log.warning(
|
|
||||||
'-- entity "{}" was not found in DB by id "{}". Skipping'.format(
|
|
||||||
ent_name, str(mongoid)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
return
|
|
||||||
|
|
||||||
# Change value if entity has set it's own
|
|
||||||
entity_value = custom_attributes[key]
|
|
||||||
if entity_value is not None:
|
|
||||||
value = entity_value
|
|
||||||
|
|
||||||
data = mongo_entity.get('data') or {}
|
|
||||||
|
|
||||||
data[key] = value
|
|
||||||
self.db_con.update_many(
|
|
||||||
{'_id': mongoid},
|
|
||||||
{'$set': {'data': data}}
|
|
||||||
)
|
|
||||||
|
|
||||||
self.log.debug(
|
|
||||||
'-- stored value "{}"'.format(value)
|
|
||||||
)
|
|
||||||
|
|
||||||
for child in entity.get('children', []):
|
|
||||||
self.update_hierarchical_attribute(child, key, value)
|
|
||||||
|
|
||||||
|
|
||||||
def register(session, plugins_presets={}):
|
|
||||||
'''Register plugin. Called when used as an plugin.'''
|
|
||||||
|
|
||||||
SyncHierarchicalAttrs(session, plugins_presets).register()
|
|
||||||
|
|
||||||
|
|
||||||
def main(arguments=None):
|
|
||||||
'''Set up logging and register action.'''
|
|
||||||
if arguments is None:
|
|
||||||
arguments = []
|
|
||||||
|
|
||||||
parser = argparse.ArgumentParser()
|
|
||||||
# Allow setting of logging level from arguments.
|
|
||||||
loggingLevels = {}
|
|
||||||
for level in (
|
|
||||||
logging.NOTSET, logging.DEBUG, logging.INFO, logging.WARNING,
|
|
||||||
logging.ERROR, logging.CRITICAL
|
|
||||||
):
|
|
||||||
loggingLevels[logging.getLevelName(level).lower()] = level
|
|
||||||
|
|
||||||
parser.add_argument(
|
|
||||||
'-v', '--verbosity',
|
|
||||||
help='Set the logging output verbosity.',
|
|
||||||
choices=loggingLevels.keys(),
|
|
||||||
default='info'
|
|
||||||
)
|
|
||||||
namespace = parser.parse_args(arguments)
|
|
||||||
|
|
||||||
# Set up basic logging
|
|
||||||
logging.basicConfig(level=loggingLevels[namespace.verbosity])
|
|
||||||
|
|
||||||
session = ftrack_api.Session()
|
|
||||||
register(session)
|
|
||||||
|
|
||||||
# Wait for events
|
|
||||||
logging.info(
|
|
||||||
'Registered actions and listening for events. Use Ctrl-C to abort.'
|
|
||||||
)
|
|
||||||
session.event_hub.wait()
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
raise SystemExit(main(sys.argv[1:]))
|
|
||||||
2327
pype/ftrack/actions/action_sync_to_avalon.py
Normal file
2327
pype/ftrack/actions/action_sync_to_avalon.py
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -1,266 +0,0 @@
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
import time
|
|
||||||
import argparse
|
|
||||||
import logging
|
|
||||||
import json
|
|
||||||
import collections
|
|
||||||
|
|
||||||
from pype.vendor import ftrack_api
|
|
||||||
from pype.ftrack import BaseAction
|
|
||||||
from pype.ftrack.lib import avalon_sync as ftracklib
|
|
||||||
from pype.vendor.ftrack_api import session as fa_session
|
|
||||||
|
|
||||||
|
|
||||||
class SyncToAvalon(BaseAction):
|
|
||||||
'''
|
|
||||||
Synchronizing data action - from Ftrack to Avalon DB
|
|
||||||
|
|
||||||
Stores all information about entity.
|
|
||||||
- Name(string) - Most important information = identifier of entity
|
|
||||||
- Parent(ObjectId) - Avalon Project Id, if entity is not project itself
|
|
||||||
- Silo(string) - Last parent except project
|
|
||||||
- Data(dictionary):
|
|
||||||
- VisualParent(ObjectId) - Avalon Id of parent asset
|
|
||||||
- Parents(array of string) - All parent names except project
|
|
||||||
- Tasks(array of string) - Tasks on asset
|
|
||||||
- FtrackId(string)
|
|
||||||
- entityType(string) - entity's type on Ftrack
|
|
||||||
* All Custom attributes in group 'Avalon' which name don't start with 'avalon_'
|
|
||||||
|
|
||||||
* These information are stored also for all parents and children entities.
|
|
||||||
|
|
||||||
Avalon ID of asset is stored to Ftrack -> Custom attribute 'avalon_mongo_id'.
|
|
||||||
- action IS NOT creating this Custom attribute if doesn't exist
|
|
||||||
- run 'Create Custom Attributes' action or do it manually (Not recommended)
|
|
||||||
|
|
||||||
If Ftrack entity already has Custom Attribute 'avalon_mongo_id' that stores ID:
|
|
||||||
- name, parents and silo are checked -> shows error if are not exact the same
|
|
||||||
- after sync it is not allowed to change names or move entities
|
|
||||||
|
|
||||||
If ID in 'avalon_mongo_id' is empty string or is not found in DB:
|
|
||||||
- tries to find entity by name
|
|
||||||
- found:
|
|
||||||
- raise error if ftrackId/visual parent/parents are not same
|
|
||||||
- not found:
|
|
||||||
- Creates asset/project
|
|
||||||
|
|
||||||
'''
|
|
||||||
|
|
||||||
#: Action identifier.
|
|
||||||
identifier = 'sync.to.avalon.local'
|
|
||||||
#: Action label.
|
|
||||||
label = "Pype Admin"
|
|
||||||
variant = '- Sync To Avalon (Local)'
|
|
||||||
#: Action description.
|
|
||||||
description = 'Send data from Ftrack to Avalon'
|
|
||||||
#: Action icon.
|
|
||||||
icon = '{}/ftrack/action_icons/PypeAdmin.svg'.format(
|
|
||||||
os.environ.get('PYPE_STATICS_SERVER', '')
|
|
||||||
)
|
|
||||||
#: roles that are allowed to register this action
|
|
||||||
role_list = ['Pypeclub']
|
|
||||||
#: Action priority
|
|
||||||
priority = 200
|
|
||||||
|
|
||||||
project_query = (
|
|
||||||
"select full_name, name, custom_attributes"
|
|
||||||
", project_schema._task_type_schema.types.name"
|
|
||||||
" from Project where full_name is \"{}\""
|
|
||||||
)
|
|
||||||
|
|
||||||
entities_query = (
|
|
||||||
"select id, name, parent_id, link, custom_attributes"
|
|
||||||
" from TypedContext where project.full_name is \"{}\""
|
|
||||||
)
|
|
||||||
|
|
||||||
# Entity type names(lowered) that won't be synchronized with their children
|
|
||||||
ignore_entity_types = ["task", "milestone"]
|
|
||||||
|
|
||||||
def __init__(self, session, plugins_presets):
|
|
||||||
super(SyncToAvalon, self).__init__(session)
|
|
||||||
# reload utils on initialize (in case of server restart)
|
|
||||||
|
|
||||||
def discover(self, session, entities, event):
|
|
||||||
''' Validation '''
|
|
||||||
for entity in entities:
|
|
||||||
if entity.entity_type.lower() not in ['task', 'assetversion']:
|
|
||||||
return True
|
|
||||||
|
|
||||||
return False
|
|
||||||
|
|
||||||
def launch(self, session, entities, event):
|
|
||||||
time_start = time.time()
|
|
||||||
message = ""
|
|
||||||
|
|
||||||
# JOB SETTINGS
|
|
||||||
userId = event['source']['user']['id']
|
|
||||||
user = session.query('User where id is ' + userId).one()
|
|
||||||
|
|
||||||
job = session.create('Job', {
|
|
||||||
'user': user,
|
|
||||||
'status': 'running',
|
|
||||||
'data': json.dumps({
|
|
||||||
'description': 'Sync Ftrack to Avalon.'
|
|
||||||
})
|
|
||||||
})
|
|
||||||
session.commit()
|
|
||||||
try:
|
|
||||||
self.log.debug("Preparing entities for synchronization")
|
|
||||||
|
|
||||||
if entities[0].entity_type.lower() == "project":
|
|
||||||
ft_project_name = entities[0]["full_name"]
|
|
||||||
else:
|
|
||||||
ft_project_name = entities[0]["project"]["full_name"]
|
|
||||||
|
|
||||||
project_entities = session.query(
|
|
||||||
self.entities_query.format(ft_project_name)
|
|
||||||
).all()
|
|
||||||
|
|
||||||
ft_project = session.query(
|
|
||||||
self.project_query.format(ft_project_name)
|
|
||||||
).one()
|
|
||||||
|
|
||||||
entities_by_id = {}
|
|
||||||
entities_by_parent = collections.defaultdict(list)
|
|
||||||
|
|
||||||
entities_by_id[ft_project["id"]] = ft_project
|
|
||||||
for ent in project_entities:
|
|
||||||
entities_by_id[ent["id"]] = ent
|
|
||||||
entities_by_parent[ent["parent_id"]].append(ent)
|
|
||||||
|
|
||||||
importable = []
|
|
||||||
for ent_info in event["data"]["selection"]:
|
|
||||||
ent = entities_by_id[ent_info["entityId"]]
|
|
||||||
for link_ent_info in ent["link"]:
|
|
||||||
link_ent = entities_by_id[link_ent_info["id"]]
|
|
||||||
if (
|
|
||||||
ent.entity_type.lower() in self.ignore_entity_types or
|
|
||||||
link_ent in importable
|
|
||||||
):
|
|
||||||
continue
|
|
||||||
|
|
||||||
importable.append(link_ent)
|
|
||||||
|
|
||||||
def add_children(parent_id):
|
|
||||||
ents = entities_by_parent[parent_id]
|
|
||||||
for ent in ents:
|
|
||||||
if ent.entity_type.lower() in self.ignore_entity_types:
|
|
||||||
continue
|
|
||||||
|
|
||||||
if ent not in importable:
|
|
||||||
importable.append(ent)
|
|
||||||
|
|
||||||
add_children(ent["id"])
|
|
||||||
|
|
||||||
# add children of selection to importable
|
|
||||||
for ent_info in event["data"]["selection"]:
|
|
||||||
add_children(ent_info["entityId"])
|
|
||||||
|
|
||||||
# Check names: REGEX in schema/duplicates - raise error if found
|
|
||||||
all_names = []
|
|
||||||
duplicates = []
|
|
||||||
|
|
||||||
for entity in importable:
|
|
||||||
ftracklib.avalon_check_name(entity)
|
|
||||||
if entity.entity_type.lower() == "project":
|
|
||||||
continue
|
|
||||||
|
|
||||||
if entity['name'] in all_names:
|
|
||||||
duplicates.append("'{}'".format(entity['name']))
|
|
||||||
else:
|
|
||||||
all_names.append(entity['name'])
|
|
||||||
|
|
||||||
if len(duplicates) > 0:
|
|
||||||
# TODO Show information to user and return False
|
|
||||||
raise ValueError(
|
|
||||||
"Entity name duplication: {}".format(", ".join(duplicates))
|
|
||||||
)
|
|
||||||
|
|
||||||
# ----- PROJECT ------
|
|
||||||
avalon_project = ftracklib.get_avalon_project(ft_project)
|
|
||||||
custom_attributes = ftracklib.get_avalon_attr(session)
|
|
||||||
|
|
||||||
# Import all entities to Avalon DB
|
|
||||||
for entity in importable:
|
|
||||||
result = ftracklib.import_to_avalon(
|
|
||||||
session=session,
|
|
||||||
entity=entity,
|
|
||||||
ft_project=ft_project,
|
|
||||||
av_project=avalon_project,
|
|
||||||
custom_attributes=custom_attributes
|
|
||||||
)
|
|
||||||
# TODO better error handling
|
|
||||||
# maybe split into critical, warnings and messages?
|
|
||||||
if 'errors' in result and len(result['errors']) > 0:
|
|
||||||
job['status'] = 'failed'
|
|
||||||
session.commit()
|
|
||||||
|
|
||||||
ftracklib.show_errors(self, event, result['errors'])
|
|
||||||
|
|
||||||
return {
|
|
||||||
'success': False,
|
|
||||||
'message': "Sync to avalon FAILED"
|
|
||||||
}
|
|
||||||
|
|
||||||
if avalon_project is None:
|
|
||||||
if 'project' in result:
|
|
||||||
avalon_project = result['project']
|
|
||||||
|
|
||||||
job['status'] = 'done'
|
|
||||||
|
|
||||||
except ValueError as ve:
|
|
||||||
# TODO remove this part!!!!
|
|
||||||
job['status'] = 'failed'
|
|
||||||
message = str(ve)
|
|
||||||
self.log.error(
|
|
||||||
'Error during syncToAvalon: {}'.format(message),
|
|
||||||
exc_info=True
|
|
||||||
)
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
job['status'] = 'failed'
|
|
||||||
exc_type, exc_obj, exc_tb = sys.exc_info()
|
|
||||||
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
|
|
||||||
log_message = "{}/{}/Line: {}".format(
|
|
||||||
exc_type, fname, exc_tb.tb_lineno
|
|
||||||
)
|
|
||||||
self.log.error(
|
|
||||||
'Error during syncToAvalon: {}'.format(log_message),
|
|
||||||
exc_info=True
|
|
||||||
)
|
|
||||||
# TODO add traceback to message and show to user
|
|
||||||
message = (
|
|
||||||
'Unexpected Error'
|
|
||||||
' - Please check Log for more information'
|
|
||||||
)
|
|
||||||
finally:
|
|
||||||
if job['status'] in ['queued', 'running']:
|
|
||||||
job['status'] = 'failed'
|
|
||||||
session.commit()
|
|
||||||
|
|
||||||
time_end = time.time()
|
|
||||||
self.log.debug("Synchronization took \"{}\"".format(
|
|
||||||
str(time_end - time_start)
|
|
||||||
))
|
|
||||||
|
|
||||||
if job["status"] != "failed":
|
|
||||||
self.log.debug("Triggering Sync hierarchical attributes")
|
|
||||||
self.trigger_action("sync.hierarchical.attrs.local", event)
|
|
||||||
|
|
||||||
if len(message) > 0:
|
|
||||||
message = "Unable to sync: {}".format(message)
|
|
||||||
return {
|
|
||||||
'success': False,
|
|
||||||
'message': message
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
'success': True,
|
|
||||||
'message': "Synchronization was successfull"
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def register(session, plugins_presets={}):
|
|
||||||
'''Register plugin. Called when used as an plugin.'''
|
|
||||||
SyncToAvalon(session, plugins_presets).register()
|
|
||||||
|
|
@ -1,383 +0,0 @@
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
import json
|
|
||||||
import argparse
|
|
||||||
import logging
|
|
||||||
import collections
|
|
||||||
|
|
||||||
from pypeapp import config
|
|
||||||
from pype.vendor import ftrack_api
|
|
||||||
from pype.ftrack import BaseAction, lib
|
|
||||||
from pype.ftrack.lib.io_nonsingleton import DbConnector
|
|
||||||
from bson.objectid import ObjectId
|
|
||||||
|
|
||||||
|
|
||||||
class SyncHierarchicalAttrs(BaseAction):
|
|
||||||
|
|
||||||
db_con = DbConnector()
|
|
||||||
ca_mongoid = lib.get_ca_mongoid()
|
|
||||||
|
|
||||||
#: Action identifier.
|
|
||||||
identifier = 'sync.hierarchical.attrs'
|
|
||||||
#: Action label.
|
|
||||||
label = "Pype Admin"
|
|
||||||
variant = '- Sync Hier Attrs (Server)'
|
|
||||||
#: Action description.
|
|
||||||
description = 'Synchronize hierarchical attributes'
|
|
||||||
#: Icon
|
|
||||||
icon = '{}/ftrack/action_icons/PypeAdmin.svg'.format(
|
|
||||||
os.environ.get(
|
|
||||||
'PYPE_STATICS_SERVER',
|
|
||||||
'http://localhost:{}'.format(
|
|
||||||
config.get_presets().get('services', {}).get(
|
|
||||||
'statics_server', {}
|
|
||||||
).get('default_port', 8021)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
def register(self):
|
|
||||||
self.session.event_hub.subscribe(
|
|
||||||
'topic=ftrack.action.discover',
|
|
||||||
self._discover
|
|
||||||
)
|
|
||||||
|
|
||||||
self.session.event_hub.subscribe(
|
|
||||||
'topic=ftrack.action.launch and data.actionIdentifier={}'.format(
|
|
||||||
self.identifier
|
|
||||||
),
|
|
||||||
self._launch
|
|
||||||
)
|
|
||||||
|
|
||||||
def discover(self, session, entities, event):
|
|
||||||
''' Validation '''
|
|
||||||
role_check = False
|
|
||||||
discover = False
|
|
||||||
role_list = ['Pypeclub', 'Administrator', 'Project Manager']
|
|
||||||
user = session.query(
|
|
||||||
'User where id is "{}"'.format(event['source']['user']['id'])
|
|
||||||
).one()
|
|
||||||
|
|
||||||
for role in user['user_security_roles']:
|
|
||||||
if role['security_role']['name'] in role_list:
|
|
||||||
role_check = True
|
|
||||||
break
|
|
||||||
|
|
||||||
if role_check is True:
|
|
||||||
for entity in entities:
|
|
||||||
context_type = entity.get('context_type', '').lower()
|
|
||||||
if (
|
|
||||||
context_type in ('show', 'task') and
|
|
||||||
entity.entity_type.lower() != 'task'
|
|
||||||
):
|
|
||||||
discover = True
|
|
||||||
break
|
|
||||||
|
|
||||||
return discover
|
|
||||||
|
|
||||||
def launch(self, session, entities, event):
|
|
||||||
self.interface_messages = {}
|
|
||||||
|
|
||||||
user = session.query(
|
|
||||||
'User where id is "{}"'.format(event['source']['user']['id'])
|
|
||||||
).one()
|
|
||||||
|
|
||||||
job = session.create('Job', {
|
|
||||||
'user': user,
|
|
||||||
'status': 'running',
|
|
||||||
'data': json.dumps({
|
|
||||||
'description': 'Sync Hierachical attributes'
|
|
||||||
})
|
|
||||||
})
|
|
||||||
session.commit()
|
|
||||||
self.log.debug('Job with id "{}" created'.format(job['id']))
|
|
||||||
|
|
||||||
process_session = ftrack_api.Session(
|
|
||||||
server_url=session.server_url,
|
|
||||||
api_key=session.api_key,
|
|
||||||
api_user=session.api_user,
|
|
||||||
auto_connect_event_hub=True
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
# Collect hierarchical attrs
|
|
||||||
self.log.debug('Collecting Hierarchical custom attributes started')
|
|
||||||
custom_attributes = {}
|
|
||||||
all_avalon_attr = process_session.query(
|
|
||||||
'CustomAttributeGroup where name is "avalon"'
|
|
||||||
).one()
|
|
||||||
|
|
||||||
error_key = (
|
|
||||||
'Hierarchical attributes with set "default" value (not allowed)'
|
|
||||||
)
|
|
||||||
|
|
||||||
for cust_attr in all_avalon_attr['custom_attribute_configurations']:
|
|
||||||
if 'avalon_' in cust_attr['key']:
|
|
||||||
continue
|
|
||||||
|
|
||||||
if not cust_attr['is_hierarchical']:
|
|
||||||
continue
|
|
||||||
|
|
||||||
if cust_attr['default']:
|
|
||||||
if error_key not in self.interface_messages:
|
|
||||||
self.interface_messages[error_key] = []
|
|
||||||
self.interface_messages[error_key].append(
|
|
||||||
cust_attr['label']
|
|
||||||
)
|
|
||||||
|
|
||||||
self.log.warning((
|
|
||||||
'Custom attribute "{}" has set default value.'
|
|
||||||
' This attribute can\'t be synchronized'
|
|
||||||
).format(cust_attr['label']))
|
|
||||||
continue
|
|
||||||
|
|
||||||
custom_attributes[cust_attr['key']] = cust_attr
|
|
||||||
|
|
||||||
self.log.debug(
|
|
||||||
'Collecting Hierarchical custom attributes has finished'
|
|
||||||
)
|
|
||||||
|
|
||||||
if not custom_attributes:
|
|
||||||
msg = 'No hierarchical attributes to sync.'
|
|
||||||
self.log.debug(msg)
|
|
||||||
return {
|
|
||||||
'success': True,
|
|
||||||
'message': msg
|
|
||||||
}
|
|
||||||
|
|
||||||
entity = entities[0]
|
|
||||||
if entity.entity_type.lower() == 'project':
|
|
||||||
project_name = entity['full_name']
|
|
||||||
else:
|
|
||||||
project_name = entity['project']['full_name']
|
|
||||||
|
|
||||||
self.db_con.install()
|
|
||||||
self.db_con.Session['AVALON_PROJECT'] = project_name
|
|
||||||
|
|
||||||
_entities = self._get_entities(event, process_session)
|
|
||||||
|
|
||||||
for entity in _entities:
|
|
||||||
self.log.debug(30*'-')
|
|
||||||
self.log.debug(
|
|
||||||
'Processing entity "{}"'.format(entity.get('name', entity))
|
|
||||||
)
|
|
||||||
|
|
||||||
ent_name = entity.get('name', entity)
|
|
||||||
if entity.entity_type.lower() == 'project':
|
|
||||||
ent_name = entity['full_name']
|
|
||||||
|
|
||||||
for key in custom_attributes:
|
|
||||||
self.log.debug(30*'*')
|
|
||||||
self.log.debug(
|
|
||||||
'Processing Custom attribute key "{}"'.format(key)
|
|
||||||
)
|
|
||||||
# check if entity has that attribute
|
|
||||||
if key not in entity['custom_attributes']:
|
|
||||||
error_key = 'Missing key on entities'
|
|
||||||
if error_key not in self.interface_messages:
|
|
||||||
self.interface_messages[error_key] = []
|
|
||||||
|
|
||||||
self.interface_messages[error_key].append(
|
|
||||||
'- key: "{}" - entity: "{}"'.format(key, ent_name)
|
|
||||||
)
|
|
||||||
|
|
||||||
self.log.error((
|
|
||||||
'- key "{}" not found on "{}"'
|
|
||||||
).format(key, entity.get('name', entity)))
|
|
||||||
continue
|
|
||||||
|
|
||||||
value = self.get_hierarchical_value(key, entity)
|
|
||||||
if value is None:
|
|
||||||
error_key = (
|
|
||||||
'Missing value for key on entity'
|
|
||||||
' and its parents (synchronization was skipped)'
|
|
||||||
)
|
|
||||||
if error_key not in self.interface_messages:
|
|
||||||
self.interface_messages[error_key] = []
|
|
||||||
|
|
||||||
self.interface_messages[error_key].append(
|
|
||||||
'- key: "{}" - entity: "{}"'.format(key, ent_name)
|
|
||||||
)
|
|
||||||
|
|
||||||
self.log.warning((
|
|
||||||
'- key "{}" not set on "{}" or its parents'
|
|
||||||
).format(key, ent_name))
|
|
||||||
continue
|
|
||||||
|
|
||||||
self.update_hierarchical_attribute(entity, key, value)
|
|
||||||
|
|
||||||
job['status'] = 'done'
|
|
||||||
session.commit()
|
|
||||||
|
|
||||||
except Exception:
|
|
||||||
self.log.error(
|
|
||||||
'Action "{}" failed'.format(self.label),
|
|
||||||
exc_info=True
|
|
||||||
)
|
|
||||||
|
|
||||||
finally:
|
|
||||||
self.db_con.uninstall()
|
|
||||||
|
|
||||||
if job['status'] in ('queued', 'running'):
|
|
||||||
job['status'] = 'failed'
|
|
||||||
session.commit()
|
|
||||||
|
|
||||||
if self.interface_messages:
|
|
||||||
self.show_interface_from_dict(
|
|
||||||
messages=self.interface_messages,
|
|
||||||
title="something went wrong",
|
|
||||||
event=event
|
|
||||||
)
|
|
||||||
|
|
||||||
return True
|
|
||||||
|
|
||||||
def get_hierarchical_value(self, key, entity):
|
|
||||||
value = entity['custom_attributes'][key]
|
|
||||||
if (
|
|
||||||
value is not None or
|
|
||||||
entity.entity_type.lower() == 'project'
|
|
||||||
):
|
|
||||||
return value
|
|
||||||
|
|
||||||
return self.get_hierarchical_value(key, entity['parent'])
|
|
||||||
|
|
||||||
def update_hierarchical_attribute(self, entity, key, value):
|
|
||||||
if (
|
|
||||||
entity['context_type'].lower() not in ('show', 'task') or
|
|
||||||
entity.entity_type.lower() == 'task'
|
|
||||||
):
|
|
||||||
return
|
|
||||||
|
|
||||||
ent_name = entity.get('name', entity)
|
|
||||||
if entity.entity_type.lower() == 'project':
|
|
||||||
ent_name = entity['full_name']
|
|
||||||
|
|
||||||
hierarchy = '/'.join(
|
|
||||||
[a['name'] for a in entity.get('ancestors', [])]
|
|
||||||
)
|
|
||||||
if hierarchy:
|
|
||||||
hierarchy = '/'.join(
|
|
||||||
[entity['project']['full_name'], hierarchy, entity['name']]
|
|
||||||
)
|
|
||||||
elif entity.entity_type.lower() == 'project':
|
|
||||||
hierarchy = entity['full_name']
|
|
||||||
else:
|
|
||||||
hierarchy = '/'.join(
|
|
||||||
[entity['project']['full_name'], entity['name']]
|
|
||||||
)
|
|
||||||
|
|
||||||
self.log.debug('- updating entity "{}"'.format(hierarchy))
|
|
||||||
|
|
||||||
# collect entity's custom attributes
|
|
||||||
custom_attributes = entity.get('custom_attributes')
|
|
||||||
if not custom_attributes:
|
|
||||||
return
|
|
||||||
|
|
||||||
mongoid = custom_attributes.get(self.ca_mongoid)
|
|
||||||
if not mongoid:
|
|
||||||
error_key = 'Missing MongoID on entities (try SyncToAvalon first)'
|
|
||||||
if error_key not in self.interface_messages:
|
|
||||||
self.interface_messages[error_key] = []
|
|
||||||
|
|
||||||
if ent_name not in self.interface_messages[error_key]:
|
|
||||||
self.interface_messages[error_key].append(ent_name)
|
|
||||||
|
|
||||||
self.log.warning(
|
|
||||||
'-- entity "{}" is not synchronized to avalon. Skipping'.format(
|
|
||||||
ent_name
|
|
||||||
)
|
|
||||||
)
|
|
||||||
return
|
|
||||||
|
|
||||||
try:
|
|
||||||
mongoid = ObjectId(mongoid)
|
|
||||||
except Exception:
|
|
||||||
error_key = 'Invalid MongoID on entities (try SyncToAvalon)'
|
|
||||||
if error_key not in self.interface_messages:
|
|
||||||
self.interface_messages[error_key] = []
|
|
||||||
|
|
||||||
if ent_name not in self.interface_messages[error_key]:
|
|
||||||
self.interface_messages[error_key].append(ent_name)
|
|
||||||
|
|
||||||
self.log.warning(
|
|
||||||
'-- entity "{}" has stored invalid MongoID. Skipping'.format(
|
|
||||||
ent_name
|
|
||||||
)
|
|
||||||
)
|
|
||||||
return
|
|
||||||
# Find entity in Mongo DB
|
|
||||||
mongo_entity = self.db_con.find_one({'_id': mongoid})
|
|
||||||
if not mongo_entity:
|
|
||||||
error_key = 'Entities not found in Avalon DB (try SyncToAvalon)'
|
|
||||||
if error_key not in self.interface_messages:
|
|
||||||
self.interface_messages[error_key] = []
|
|
||||||
|
|
||||||
if ent_name not in self.interface_messages[error_key]:
|
|
||||||
self.interface_messages[error_key].append(ent_name)
|
|
||||||
|
|
||||||
self.log.warning(
|
|
||||||
'-- entity "{}" was not found in DB by id "{}". Skipping'.format(
|
|
||||||
ent_name, str(mongoid)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
return
|
|
||||||
|
|
||||||
# Change value if entity has set it's own
|
|
||||||
entity_value = custom_attributes[key]
|
|
||||||
if entity_value is not None:
|
|
||||||
value = entity_value
|
|
||||||
|
|
||||||
data = mongo_entity.get('data') or {}
|
|
||||||
|
|
||||||
data[key] = value
|
|
||||||
self.db_con.update_many(
|
|
||||||
{'_id': mongoid},
|
|
||||||
{'$set': {'data': data}}
|
|
||||||
)
|
|
||||||
|
|
||||||
for child in entity.get('children', []):
|
|
||||||
self.update_hierarchical_attribute(child, key, value)
|
|
||||||
|
|
||||||
|
|
||||||
def register(session, plugins_presets):
|
|
||||||
'''Register plugin. Called when used as an plugin.'''
|
|
||||||
|
|
||||||
SyncHierarchicalAttrs(session, plugins_presets).register()
|
|
||||||
|
|
||||||
|
|
||||||
def main(arguments=None):
|
|
||||||
'''Set up logging and register action.'''
|
|
||||||
if arguments is None:
|
|
||||||
arguments = []
|
|
||||||
|
|
||||||
parser = argparse.ArgumentParser()
|
|
||||||
# Allow setting of logging level from arguments.
|
|
||||||
loggingLevels = {}
|
|
||||||
for level in (
|
|
||||||
logging.NOTSET, logging.DEBUG, logging.INFO, logging.WARNING,
|
|
||||||
logging.ERROR, logging.CRITICAL
|
|
||||||
):
|
|
||||||
loggingLevels[logging.getLevelName(level).lower()] = level
|
|
||||||
|
|
||||||
parser.add_argument(
|
|
||||||
'-v', '--verbosity',
|
|
||||||
help='Set the logging output verbosity.',
|
|
||||||
choices=loggingLevels.keys(),
|
|
||||||
default='info'
|
|
||||||
)
|
|
||||||
namespace = parser.parse_args(arguments)
|
|
||||||
|
|
||||||
# Set up basic logging
|
|
||||||
logging.basicConfig(level=loggingLevels[namespace.verbosity])
|
|
||||||
|
|
||||||
session = ftrack_api.Session()
|
|
||||||
register(session)
|
|
||||||
|
|
||||||
# Wait for events
|
|
||||||
logging.info(
|
|
||||||
'Registered actions and listening for events. Use Ctrl-C to abort.'
|
|
||||||
)
|
|
||||||
session.event_hub.wait()
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
raise SystemExit(main(sys.argv[1:]))
|
|
||||||
File diff suppressed because it is too large
Load diff
|
|
@ -50,6 +50,19 @@ class DbConnector(object):
|
||||||
self._database = None
|
self._database = None
|
||||||
self._is_installed = False
|
self._is_installed = False
|
||||||
|
|
||||||
|
def __getitem__(self, key):
|
||||||
|
# gives direct access to collection withou setting `active_table`
|
||||||
|
return self._database[key]
|
||||||
|
|
||||||
|
def __getattribute__(self, attr):
|
||||||
|
# not all methods of PyMongo database are implemented with this it is
|
||||||
|
# possible to use them too
|
||||||
|
try:
|
||||||
|
return super(DbConnector, self).__getattribute__(attr)
|
||||||
|
except AttributeError:
|
||||||
|
cur_proj = self.Session["AVALON_PROJECT"]
|
||||||
|
return self._database[cur_proj].__getattribute__(attr)
|
||||||
|
|
||||||
def install(self):
|
def install(self):
|
||||||
"""Establish a persistent connection to the database"""
|
"""Establish a persistent connection to the database"""
|
||||||
if self._is_installed:
|
if self._is_installed:
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue