Merged in feature/PYPE-460_hier_attr_sync_enhancement (pull request #249)

Feature/PYPE-460 hier attr sync enhancement

Approved-by: Milan Kolar <milan@orbi.tools>
This commit is contained in:
Jakub Trllo 2019-08-04 20:33:11 +00:00 committed by Milan Kolar
commit 5d1069ca85
5 changed files with 338 additions and 54 deletions

View file

@ -28,7 +28,7 @@ class SyncHierarchicalAttrs(BaseAction):
)
#: roles that are allowed to register this action
role_list = ['Administrator']
role_list = ['Pypeclub', 'Administrator', 'Project Manager']
def discover(self, session, entities, event):
''' Validation '''
@ -41,6 +41,7 @@ class SyncHierarchicalAttrs(BaseAction):
return False
def launch(self, session, entities, event):
self.interface_messages = {}
user = session.query(
'User where id is "{}"'.format(event['source']['user']['id'])
).one()
@ -53,13 +54,27 @@ class SyncHierarchicalAttrs(BaseAction):
})
})
session.commit()
self.log.debug('Job with id "{}" created'.format(job['id']))
process_session = ftrack_api.Session(
server_url=session.server_url,
api_key=session.api_key,
api_user=session.api_user,
auto_connect_event_hub=True
)
try:
# Collect hierarchical attrs
self.log.debug('Collecting Hierarchical custom attributes started')
custom_attributes = {}
all_avalon_attr = session.query(
all_avalon_attr = process_session.query(
'CustomAttributeGroup where name is "avalon"'
).one()
error_key = (
'Hierarchical attributes with set "default" value (not allowed)'
)
for cust_attr in all_avalon_attr['custom_attribute_configurations']:
if 'avalon_' in cust_attr['key']:
continue
@ -68,6 +83,12 @@ class SyncHierarchicalAttrs(BaseAction):
continue
if cust_attr['default']:
if error_key not in self.interface_messages:
self.interface_messages[error_key] = []
self.interface_messages[error_key].append(
cust_attr['label']
)
self.log.warning((
'Custom attribute "{}" has set default value.'
' This attribute can\'t be synchronized'
@ -76,6 +97,10 @@ class SyncHierarchicalAttrs(BaseAction):
custom_attributes[cust_attr['key']] = cust_attr
self.log.debug(
'Collecting Hierarchical custom attributes has finished'
)
if not custom_attributes:
msg = 'No hierarchical attributes to sync.'
self.log.debug(msg)
@ -93,28 +118,61 @@ class SyncHierarchicalAttrs(BaseAction):
self.db_con.install()
self.db_con.Session['AVALON_PROJECT'] = project_name
for entity in entities:
_entities = self._get_entities(event, process_session)
for entity in _entities:
self.log.debug(30*'-')
self.log.debug(
'Processing entity "{}"'.format(entity.get('name', entity))
)
ent_name = entity.get('name', entity)
if entity.entity_type.lower() == 'project':
ent_name = entity['full_name']
for key in custom_attributes:
self.log.debug(30*'*')
self.log.debug(
'Processing Custom attribute key "{}"'.format(key)
)
# check if entity has that attribute
if key not in entity['custom_attributes']:
self.log.debug(
'Hierachical attribute "{}" not found on "{}"'.format(
key, entity.get('name', entity)
)
error_key = 'Missing key on entities'
if error_key not in self.interface_messages:
self.interface_messages[error_key] = []
self.interface_messages[error_key].append(
'- key: "{}" - entity: "{}"'.format(key, ent_name)
)
self.log.error((
'- key "{}" not found on "{}"'
).format(key, ent_name))
continue
value = self.get_hierarchical_value(key, entity)
if value is None:
self.log.warning(
'Hierarchical attribute "{}" not set on "{}"'.format(
key, entity.get('name', entity)
)
error_key = (
'Missing value for key on entity'
' and its parents (synchronization was skipped)'
)
if error_key not in self.interface_messages:
self.interface_messages[error_key] = []
self.interface_messages[error_key].append(
'- key: "{}" - entity: "{}"'.format(key, ent_name)
)
self.log.warning((
'- key "{}" not set on "{}" or its parents'
).format(key, ent_name))
continue
self.update_hierarchical_attribute(entity, key, value)
job['status'] = 'done'
session.commit()
except Exception:
self.log.error(
'Action "{}" failed'.format(self.label),
@ -127,6 +185,8 @@ class SyncHierarchicalAttrs(BaseAction):
if job['status'] in ('queued', 'running'):
job['status'] = 'failed'
session.commit()
if self.interface_messages:
self.show_interface_from_dict(self.interface_messages, event)
return True
@ -146,6 +206,27 @@ class SyncHierarchicalAttrs(BaseAction):
entity.entity_type.lower() == 'task'
):
return
ent_name = entity.get('name', entity)
if entity.entity_type.lower() == 'project':
ent_name = entity['full_name']
hierarchy = '/'.join(
[a['name'] for a in entity.get('ancestors', [])]
)
if hierarchy:
hierarchy = '/'.join(
[entity['project']['full_name'], hierarchy, entity['name']]
)
elif entity.entity_type.lower() == 'project':
hierarchy = entity['full_name']
else:
hierarchy = '/'.join(
[entity['project']['full_name'], entity['name']]
)
self.log.debug('- updating entity "{}"'.format(hierarchy))
# collect entity's custom attributes
custom_attributes = entity.get('custom_attributes')
if not custom_attributes:
@ -153,24 +234,49 @@ class SyncHierarchicalAttrs(BaseAction):
mongoid = custom_attributes.get(self.ca_mongoid)
if not mongoid:
self.log.debug('Entity "{}" is not synchronized to avalon.'.format(
entity.get('name', entity)
))
error_key = 'Missing MongoID on entities (try SyncToAvalon first)'
if error_key not in self.interface_messages:
self.interface_messages[error_key] = []
if ent_name not in self.interface_messages[error_key]:
self.interface_messages[error_key].append(ent_name)
self.log.warning(
'-- entity "{}" is not synchronized to avalon. Skipping'.format(
ent_name
)
)
return
try:
mongoid = ObjectId(mongoid)
except Exception:
self.log.warning('Entity "{}" has stored invalid MongoID.'.format(
entity.get('name', entity)
))
error_key = 'Invalid MongoID on entities (try SyncToAvalon)'
if error_key not in self.interface_messages:
self.interface_messages[error_key] = []
if ent_name not in self.interface_messages[error_key]:
self.interface_messages[error_key].append(ent_name)
self.log.warning(
'-- entity "{}" has stored invalid MongoID. Skipping'.format(
ent_name
)
)
return
# Find entity in Mongo DB
mongo_entity = self.db_con.find_one({'_id': mongoid})
if not mongo_entity:
error_key = 'Entities not found in Avalon DB (try SyncToAvalon)'
if error_key not in self.interface_messages:
self.interface_messages[error_key] = []
if ent_name not in self.interface_messages[error_key]:
self.interface_messages[error_key].append(ent_name)
self.log.warning(
'Entity "{}" is not synchronized to avalon.'.format(
entity.get('name', entity)
'-- entity "{}" was not found in DB by id "{}". Skipping'.format(
ent_name, str(mongoid)
)
)
return
@ -188,6 +294,10 @@ class SyncHierarchicalAttrs(BaseAction):
{'$set': {'data': data}}
)
self.log.debug(
'-- stored value "{}"'.format(value)
)
for child in entity.get('children', []):
self.update_hierarchical_attribute(child, key, value)

View file

@ -61,7 +61,7 @@ class SyncHierarchicalAttrs(BaseAction):
if role['security_role']['name'] in role_list:
role_check = True
break
print(self.icon)
if role_check is True:
for entity in entities:
context_type = entity.get('context_type', '').lower()
@ -75,6 +75,8 @@ class SyncHierarchicalAttrs(BaseAction):
return discover
def launch(self, session, entities, event):
self.interface_messages = {}
user = session.query(
'User where id is "{}"'.format(event['source']['user']['id'])
).one()
@ -87,13 +89,26 @@ class SyncHierarchicalAttrs(BaseAction):
})
})
session.commit()
self.log.debug('Job with id "{}" created'.format(job['id']))
process_session = ftrack_api.Session(
server_url=session.server_url,
api_key=session.api_key,
api_user=session.api_user,
auto_connect_event_hub=True
)
try:
# Collect hierarchical attrs
self.log.debug('Collecting Hierarchical custom attributes started')
custom_attributes = {}
all_avalon_attr = session.query(
all_avalon_attr = process_session.query(
'CustomAttributeGroup where name is "avalon"'
).one()
error_key = (
'Hierarchical attributes with set "default" value (not allowed)'
)
for cust_attr in all_avalon_attr['custom_attribute_configurations']:
if 'avalon_' in cust_attr['key']:
continue
@ -102,6 +117,12 @@ class SyncHierarchicalAttrs(BaseAction):
continue
if cust_attr['default']:
if error_key not in self.interface_messages:
self.interface_messages[error_key] = []
self.interface_messages[error_key].append(
cust_attr['label']
)
self.log.warning((
'Custom attribute "{}" has set default value.'
' This attribute can\'t be synchronized'
@ -110,6 +131,10 @@ class SyncHierarchicalAttrs(BaseAction):
custom_attributes[cust_attr['key']] = cust_attr
self.log.debug(
'Collecting Hierarchical custom attributes has finished'
)
if not custom_attributes:
msg = 'No hierarchical attributes to sync.'
self.log.debug(msg)
@ -127,28 +152,61 @@ class SyncHierarchicalAttrs(BaseAction):
self.db_con.install()
self.db_con.Session['AVALON_PROJECT'] = project_name
for entity in entities:
_entities = self._get_entities(event, process_session)
for entity in _entities:
self.log.debug(30*'-')
self.log.debug(
'Processing entity "{}"'.format(entity.get('name', entity))
)
ent_name = entity.get('name', entity)
if entity.entity_type.lower() == 'project':
ent_name = entity['full_name']
for key in custom_attributes:
self.log.debug(30*'*')
self.log.debug(
'Processing Custom attribute key "{}"'.format(key)
)
# check if entity has that attribute
if key not in entity['custom_attributes']:
self.log.debug(
'Hierachical attribute "{}" not found on "{}"'.format(
key, entity.get('name', entity)
)
error_key = 'Missing key on entities'
if error_key not in self.interface_messages:
self.interface_messages[error_key] = []
self.interface_messages[error_key].append(
'- key: "{}" - entity: "{}"'.format(key, ent_name)
)
self.log.error((
'- key "{}" not found on "{}"'
).format(key, entity.get('name', entity)))
continue
value = self.get_hierarchical_value(key, entity)
if value is None:
self.log.warning(
'Hierarchical attribute "{}" not set on "{}"'.format(
key, entity.get('name', entity)
)
error_key = (
'Missing value for key on entity'
' and its parents (synchronization was skipped)'
)
if error_key not in self.interface_messages:
self.interface_messages[error_key] = []
self.interface_messages[error_key].append(
'- key: "{}" - entity: "{}"'.format(key, ent_name)
)
self.log.warning((
'- key "{}" not set on "{}" or its parents'
).format(key, ent_name))
continue
self.update_hierarchical_attribute(entity, key, value)
job['status'] = 'done'
session.commit()
except Exception:
self.log.error(
'Action "{}" failed'.format(self.label),
@ -161,6 +219,9 @@ class SyncHierarchicalAttrs(BaseAction):
if job['status'] in ('queued', 'running'):
job['status'] = 'failed'
session.commit()
if self.interface_messages:
self.show_interface_from_dict(self.interface_messages, event)
return True
@ -180,6 +241,27 @@ class SyncHierarchicalAttrs(BaseAction):
entity.entity_type.lower() == 'task'
):
return
ent_name = entity.get('name', entity)
if entity.entity_type.lower() == 'project':
ent_name = entity['full_name']
hierarchy = '/'.join(
[a['name'] for a in entity.get('ancestors', [])]
)
if hierarchy:
hierarchy = '/'.join(
[entity['project']['full_name'], hierarchy, entity['name']]
)
elif entity.entity_type.lower() == 'project':
hierarchy = entity['full_name']
else:
hierarchy = '/'.join(
[entity['project']['full_name'], entity['name']]
)
self.log.debug('- updating entity "{}"'.format(hierarchy))
# collect entity's custom attributes
custom_attributes = entity.get('custom_attributes')
if not custom_attributes:
@ -187,24 +269,49 @@ class SyncHierarchicalAttrs(BaseAction):
mongoid = custom_attributes.get(self.ca_mongoid)
if not mongoid:
self.log.debug('Entity "{}" is not synchronized to avalon.'.format(
entity.get('name', entity)
))
error_key = 'Missing MongoID on entities (try SyncToAvalon first)'
if error_key not in self.interface_messages:
self.interface_messages[error_key] = []
if ent_name not in self.interface_messages[error_key]:
self.interface_messages[error_key].append(ent_name)
self.log.warning(
'-- entity "{}" is not synchronized to avalon. Skipping'.format(
ent_name
)
)
return
try:
mongoid = ObjectId(mongoid)
except Exception:
self.log.warning('Entity "{}" has stored invalid MongoID.'.format(
entity.get('name', entity)
))
error_key = 'Invalid MongoID on entities (try SyncToAvalon)'
if error_key not in self.interface_messages:
self.interface_messages[error_key] = []
if ent_name not in self.interface_messages[error_key]:
self.interface_messages[error_key].append(ent_name)
self.log.warning(
'-- entity "{}" has stored invalid MongoID. Skipping'.format(
ent_name
)
)
return
# Find entity in Mongo DB
mongo_entity = self.db_con.find_one({'_id': mongoid})
if not mongo_entity:
error_key = 'Entities not found in Avalon DB (try SyncToAvalon)'
if error_key not in self.interface_messages:
self.interface_messages[error_key] = []
if ent_name not in self.interface_messages[error_key]:
self.interface_messages[error_key].append(ent_name)
self.log.warning(
'Entity "{}" is not synchronized to avalon.'.format(
entity.get('name', entity)
'-- entity "{}" was not found in DB by id "{}". Skipping'.format(
ent_name, str(mongoid)
)
)
return

View file

@ -112,7 +112,7 @@ class Sync_to_Avalon(BaseEvent):
{'type': 'label', 'value': '# Fatal Error'},
{'type': 'label', 'value': '<p>{}</p>'.format(ftrack_message)}
]
self.show_interface(event, items, title)
self.show_interface(items, title, event=event)
self.log.error('Fatal error during sync: {}'.format(message))
return

View file

@ -588,4 +588,4 @@ def show_errors(obj, event, errors):
obj.log.error(
'{}: {}'.format(key, message)
)
obj.show_interface(event, items, title)
obj.show_interface(items, title, event=event)

View file

@ -194,7 +194,6 @@ class BaseHandler(object):
def _translate_event(self, session, event):
'''Return *event* translated structure to be used with the API.'''
'''Return *event* translated structure to be used with the API.'''
_entities = event['data'].get('entities_object', None)
if (
_entities is None or
@ -209,25 +208,28 @@ class BaseHandler(object):
event
]
def _get_entities(self, event):
self.session._local_cache.clear()
selection = event['data'].get('selection', [])
def _get_entities(self, event, session=None):
if session is None:
session = self.session
session._local_cache.clear()
selection = event['data'].get('selection') or []
_entities = []
for entity in selection:
_entities.append(
self.session.get(
self._get_entity_type(entity),
entity.get('entityId')
)
)
_entities.append(session.get(
self._get_entity_type(entity, session),
entity.get('entityId')
))
event['data']['entities_object'] = _entities
return _entities
def _get_entity_type(self, entity):
def _get_entity_type(self, entity, session=None):
'''Return translated entity type tht can be used with API.'''
# Get entity type and make sure it is lower cased. Most places except
# the component tab in the Sidebar will use lower case notation.
entity_type = entity.get('entityType').replace('_', '').lower()
if session is None:
session = self.session
for schema in self.session.schemas:
alias_for = schema.get('alias_for')
@ -430,12 +432,47 @@ class BaseHandler(object):
on_error='ignore'
)
def show_interface(self, event, items, title=''):
def show_interface(
self, items, title='',
event=None, user=None, username=None, user_id=None
):
"""
Shows interface to user who triggered event
Shows interface to user
- to identify user must be entered one of args:
event, user, username, user_id
- 'items' must be list containing Ftrack interface items
"""
user_id = event['source']['user']['id']
if not any([event, user, username, user_id]):
raise TypeError((
'Missing argument `show_interface` requires one of args:'
' event (ftrack_api Event object),'
' user (ftrack_api User object)'
' username (string) or user_id (string)'
))
if event:
user_id = event['source']['user']['id']
elif user:
user_id = user['id']
else:
if user_id:
key = 'id'
value = user_id
else:
key = 'username'
value = username
user = self.session.query(
'User where {} is "{}"'.format(key, value)
).first()
if not user:
raise TypeError((
'Ftrack user with {} "{}" was not found!'.format(key, value)
))
user_id = user['id']
target = (
'applicationId=ftrack.client.web and user.id="{0}"'
).format(user_id)
@ -452,3 +489,33 @@ class BaseHandler(object):
),
on_error='ignore'
)
def show_interface_from_dict(
self, messages, event=None, user=None, username=None, user_id=None
):
if not messages:
self.log.debug("No messages to show! (messages dict is empty)")
return
items = []
title = 'Errors during mirroring'
splitter = {'type': 'label', 'value': '---'}
first = True
for key, value in messages.items():
if not first:
items.append(splitter)
else:
first = False
subtitle = {'type': 'label', 'value':'<h3>{}</h3>'.format(key)}
items.append(subtitle)
if isinstance(value, list):
for item in value:
message = {
'type': 'label', 'value': '<p>{}</p>'.format(item)
}
items.append(message)
else:
message = {'type': 'label', 'value': '<p>{}</p>'.format(value)}
items.append(message)
self.show_interface(items, title, event, user, username, user_id)